From d0490a4150cc6e98d8d3a72eeb86f919b32089d5 Mon Sep 17 00:00:00 2001 From: Kyle McCullough Date: Mon, 21 Oct 2024 16:34:03 -0700 Subject: [PATCH] vendor plugin sdk Signed-off-by: Kyle McCullough --- .golangci.yml | 3 + client/account.go | 6 +- client/aws_sdk.go | 3 +- client/client.go | 19 +- client/client_test.go | 3 +- client/columns.go | 3 +- client/helpers.go | 5 +- client/helpers_test.go | 5 +- client/mocks/codepipeline.go | 60 ++ client/mocks/ec2.go | 30 + client/mocks/ecr.go | 60 ++ client/mocks/elasticloadbalancingv2.go | 60 ++ client/mocks/glue.go | 120 +++ client/mocks/iot.go | 30 + client/mocks/lambda.go | 30 + client/mocks/s3control.go | 30 + client/mocks/s3manager.go | 2 +- client/mocks/sagemaker.go | 60 ++ client/multiplexers.go | 3 +- client/organizations.go | 8 +- client/organizations_test.go | 8 +- client/resolvers.go | 5 +- client/resolvers_test.go | 7 +- client/services.go | 3 +- client/services/codepipeline.go | 2 + client/services/ec2.go | 1 + client/services/ecr.go | 2 + client/services/elasticloadbalancingv2.go | 2 + client/services/glue.go | 4 + client/services/iot.go | 1 + client/services/lambda.go | 1 + client/services/s3control.go | 1 + client/services/s3manager.go | 1 + client/services/sagemaker.go | 2 + client/spec/gen/main.go | 3 +- client/spec/schema.json | 2 +- client/spec/spec.go | 3 +- client/testing.go | 13 +- client/transformers.go | 5 +- docs/tables/aws_athena_data_catalogs.md | 5 +- docs/tables/aws_backup_vaults.md | 4 +- docs/tables/aws_batch_compute_environments.md | 1 + docs/tables/aws_eks_clusters.md | 1 + docs/tables/aws_glue_connections.md | 1 + docs/tables/aws_glue_database_tables.md | 1 + docs/tables/aws_glue_job_runs.md | 2 + docs/tables/aws_glue_jobs.md | 1 + ...s_lambda_function_event_source_mappings.md | 3 + docs/tables/aws_s3_access_grant_instances.md | 4 +- go.mod | 44 +- go.sum | 12 +- main.go | 5 +- plugin-sdk/caser/caser.go | 214 +++++ plugin-sdk/caser/caser_test.go | 203 +++++ plugin-sdk/caser/initialisms.go | 57 ++ plugin-sdk/docs/generator.go | 138 +++ plugin-sdk/docs/generator_test.go | 164 ++++ plugin-sdk/docs/json.go | 64 ++ plugin-sdk/docs/markdown.go | 96 ++ .../docs/templates/all_tables.md.go.tpl | 5 + .../docs/templates/all_tables_entry.md.go.tpl | 5 + plugin-sdk/docs/templates/table.md.go.tpl | 44 + .../TestGeneratePluginDocs-JSON-__tables.json | 114 +++ .../TestGeneratePluginDocs-Markdown-README.md | 10 + ...tePluginDocs-Markdown-incremental_table.md | 16 + ...Docs-Markdown-relation_relation_table_a.md | 17 + ...Docs-Markdown-relation_relation_table_b.md | 17 + ...eratePluginDocs-Markdown-relation_table.md | 21 + ...tGeneratePluginDocs-Markdown-test_table.md | 25 + plugin-sdk/faker/faker.go | 201 +++++ plugin-sdk/faker/faker_test.go | 99 +++ plugin-sdk/faker/options.go | 17 + plugin-sdk/glob/LICENSE | 21 + plugin-sdk/glob/README.md | 3 + plugin-sdk/glob/glob.go | 70 ++ plugin-sdk/glob/glob_test.go | 105 +++ plugin-sdk/helpers/grpczerolog/grpc.go | 29 + plugin-sdk/helpers/slice.go | 36 + plugin-sdk/helpers/slice_test.go | 28 + plugin-sdk/internal/batch/cap.go | 72 ++ plugin-sdk/internal/batch/slice.go | 145 +++ plugin-sdk/internal/batch/slice_test.go | 134 +++ plugin-sdk/internal/clients/state/v3/state.go | 234 +++++ .../internal/clients/state/v3/state_test.go | 3 + plugin-sdk/internal/memdb/memdb.go | 330 +++++++ plugin-sdk/internal/memdb/memdb_test.go | 80 ++ .../servers/destination/v0/destinations.go | 265 ++++++ .../servers/destination/v0/schemav2tov3.go | 249 ++++++ .../servers/destination/v1/convert.go | 33 + .../destination/v1/destination_test.go | 160 ++++ .../servers/destination/v1/destinations.go | 241 +++++ .../servers/discovery/v0/discovery.go | 16 + .../servers/discovery/v0/discovery_test.go | 28 + .../servers/discovery/v1/discovery.go | 16 + .../servers/discovery/v1/discovery_test.go | 28 + .../internal/servers/plugin/v3/plugin.go | 401 +++++++++ .../internal/servers/plugin/v3/plugin_test.go | 198 +++++ plugin-sdk/message/sync_message.go | 124 +++ plugin-sdk/message/write_message.go | 172 ++++ plugin-sdk/plugin/diff.go | 51 ++ plugin-sdk/plugin/meta.go | 10 + plugin-sdk/plugin/nulls.go | 108 +++ plugin-sdk/plugin/nulls_test.go | 102 +++ plugin-sdk/plugin/options.go | 68 ++ plugin-sdk/plugin/plugin.go | 257 ++++++ plugin-sdk/plugin/plugin_destination.go | 55 ++ plugin-sdk/plugin/plugin_package.go | 84 ++ plugin-sdk/plugin/plugin_read.go | 34 + plugin-sdk/plugin/plugin_source.go | 100 +++ plugin-sdk/plugin/plugin_test.go | 105 +++ plugin-sdk/plugin/plugin_testconn.go | 46 + plugin-sdk/plugin/sort.go | 36 + plugin-sdk/plugin/testing.go | 12 + plugin-sdk/plugin/testing_validation.go | 18 + plugin-sdk/plugin/testing_write.go | 207 +++++ plugin-sdk/plugin/testing_write_delete.go | 302 +++++++ plugin-sdk/plugin/testing_write_insert.go | 142 +++ plugin-sdk/plugin/testing_write_migrate.go | 364 ++++++++ plugin-sdk/plugin/testing_write_upsert.go | 231 +++++ plugin-sdk/plugin/validate.go | 47 + plugin-sdk/premium/monitor.go | 108 +++ plugin-sdk/premium/monitor_test.go | 81 ++ plugin-sdk/premium/offline.go | 161 ++++ plugin-sdk/premium/offline.key | 1 + plugin-sdk/premium/offline_test.go | 165 ++++ plugin-sdk/premium/tables.go | 25 + plugin-sdk/premium/tables_test.go | 51 ++ plugin-sdk/premium/usage.go | 578 ++++++++++++ plugin-sdk/premium/usage_test.go | 829 ++++++++++++++++++ plugin-sdk/scalar/LICENSE | 23 + plugin-sdk/scalar/README.md | 4 + plugin-sdk/scalar/binary.go | 96 ++ plugin-sdk/scalar/binary_test.go | 33 + plugin-sdk/scalar/bool.go | 87 ++ plugin-sdk/scalar/bool_test.go | 39 + plugin-sdk/scalar/convert.go | 185 ++++ plugin-sdk/scalar/date32.go | 107 +++ plugin-sdk/scalar/date32_test.go | 33 + plugin-sdk/scalar/date64.go | 107 +++ plugin-sdk/scalar/date64_test.go | 31 + plugin-sdk/scalar/decimal.go | 337 +++++++ plugin-sdk/scalar/decimal_test.go | 126 +++ plugin-sdk/scalar/duration.go | 62 ++ plugin-sdk/scalar/duration_test.go | 23 + plugin-sdk/scalar/errors.go | 37 + plugin-sdk/scalar/float.go | 231 +++++ plugin-sdk/scalar/float_test.go | 170 ++++ plugin-sdk/scalar/inet.go | 150 ++++ plugin-sdk/scalar/inet_test.go | 118 +++ plugin-sdk/scalar/int.go | 253 ++++++ plugin-sdk/scalar/int_test.go | 106 +++ plugin-sdk/scalar/interval.go | 264 ++++++ plugin-sdk/scalar/json.go | 171 ++++ plugin-sdk/scalar/json_test.go | 60 ++ plugin-sdk/scalar/list.go | 167 ++++ plugin-sdk/scalar/list_test.go | 72 ++ plugin-sdk/scalar/mac.go | 92 ++ plugin-sdk/scalar/mac_test.go | 48 + plugin-sdk/scalar/scalar.go | 279 ++++++ plugin-sdk/scalar/scalar_test.go | 158 ++++ plugin-sdk/scalar/string.go | 118 +++ plugin-sdk/scalar/string_test.go | 26 + plugin-sdk/scalar/struct.go | 118 +++ plugin-sdk/scalar/struct_test.go | 147 ++++ plugin-sdk/scalar/time.go | 96 ++ plugin-sdk/scalar/timestamp.go | 151 ++++ plugin-sdk/scalar/timestamp_test.go | 143 +++ plugin-sdk/scalar/type_test.go | 10 + plugin-sdk/scalar/uint.go | 248 ++++++ plugin-sdk/scalar/uint_test.go | 96 ++ plugin-sdk/scalar/uuid.go | 123 +++ plugin-sdk/scalar/uuid_test.go | 82 ++ plugin-sdk/scheduler/batch.go | 222 +++++ plugin-sdk/scheduler/benchmark_test.go | 1 + plugin-sdk/scheduler/benchmark_test.go.backup | 429 +++++++++ plugin-sdk/scheduler/metrics.go | 247 ++++++ plugin-sdk/scheduler/metrics_test.go | 37 + plugin-sdk/scheduler/scheduler.go | 350 ++++++++ plugin-sdk/scheduler/scheduler_dfs.go | 217 +++++ plugin-sdk/scheduler/scheduler_round_robin.go | 76 ++ .../scheduler/scheduler_round_robin_test.go | 65 ++ plugin-sdk/scheduler/scheduler_shuffle.go | 76 ++ plugin-sdk/scheduler/scheduler_test.go | 489 +++++++++++ plugin-sdk/scheduler/strategy.go | 101 +++ plugin-sdk/scheduler/strategy_test.go | 74 ++ plugin-sdk/schema/arrow.go | 40 + plugin-sdk/schema/arrow_test.go | 33 + plugin-sdk/schema/column.go | 189 ++++ plugin-sdk/schema/convert.go | 202 +++++ plugin-sdk/schema/errors.go | 44 + plugin-sdk/schema/meta.go | 61 ++ plugin-sdk/schema/resolvers.go | 26 + plugin-sdk/schema/resource.go | 155 ++++ plugin-sdk/schema/table.go | 627 +++++++++++++ plugin-sdk/schema/table_test.go | 767 ++++++++++++++++ plugin-sdk/schema/testdata.go | 497 +++++++++++ plugin-sdk/schema/testdata_test.go | 54 ++ plugin-sdk/schema/validators.go | 54 ++ plugin-sdk/schema/validators_test.go | 57 ++ plugin-sdk/serve/constants.go | 12 + plugin-sdk/serve/destination_v1_test.go | 183 ++++ plugin-sdk/serve/docs.go | 56 ++ plugin-sdk/serve/docs_test.go | 22 + plugin-sdk/serve/enum.go | 45 + plugin-sdk/serve/env.go | 10 + plugin-sdk/serve/info.go | 26 + plugin-sdk/serve/opentelemetry.go | 165 ++++ plugin-sdk/serve/opentelemetry_test.go | 22 + plugin-sdk/serve/package.go | 458 ++++++++++ plugin-sdk/serve/package_test.go | 366 ++++++++ plugin-sdk/serve/plugin.go | 255 ++++++ plugin-sdk/serve/plugin_test.go | 189 ++++ plugin-sdk/serve/state_test.go | 159 ++++ .../testdata/destination_spec_schema.json | 15 + plugin-sdk/serve/testdata/memdbtables.json | 61 ++ plugin-sdk/serve/testdata/message.txt | 2 + .../serve/testdata/source_spec_schema.json | 15 + plugin-sdk/state/client.go | 109 +++ plugin-sdk/transformers/ignore_in_tests.go | 11 + plugin-sdk/transformers/name.go | 29 + plugin-sdk/transformers/options.go | 70 ++ plugin-sdk/transformers/resolver.go | 15 + plugin-sdk/transformers/struct.go | 236 +++++ plugin-sdk/transformers/struct_test.go | 483 ++++++++++ plugin-sdk/transformers/tables.go | 47 + plugin-sdk/transformers/type.go | 72 ++ plugin-sdk/types/extensions.go | 15 + plugin-sdk/types/extensions_test.go | 145 +++ plugin-sdk/types/inet.go | 249 ++++++ plugin-sdk/types/inet_test.go | 66 ++ plugin-sdk/types/json.go | 243 +++++ plugin-sdk/types/json_test.go | 427 +++++++++ plugin-sdk/types/mac.go | 234 +++++ plugin-sdk/types/mac_test.go | 64 ++ plugin-sdk/types/register.go | 29 + plugin-sdk/types/uuid.go | 235 +++++ plugin-sdk/types/uuid_test.go | 56 ++ plugin-sdk/writers/batchwriter/batchwriter.go | 368 ++++++++ .../writers/batchwriter/batchwriter_test.go | 247 ++++++ .../writers/batchwriter/unimplemented.go | 27 + .../writers/batchwriter/unimplemented_test.go | 20 + .../mixedbatchwriter/mixedbatchwriter.go | 256 ++++++ .../mixedbatchwriter/mixedbatchwriter_test.go | 326 +++++++ .../writers/mixedbatchwriter/unimplemented.go | 27 + .../mixedbatchwriter/unimplemented_test.go | 20 + plugin-sdk/writers/msgtype.go | 31 + .../streamingbatchwriter/mocktimer_test.go | 39 + .../streamingbatchwriter.go | 450 ++++++++++ .../streamingbatchwriter_test.go | 415 +++++++++ .../streamingbatchwriter/unimplemented.go | 38 + .../unimplemented_test.go | 20 + plugin-sdk/writers/ticker.go | 38 + plugin-sdk/writers/writers.go | 11 + plugin-sdk/writers/writers_test.go | 219 +++++ resources/plugin/client.go | 15 +- resources/plugin/plugin.go | 11 +- resources/plugin/tables.go | 6 +- .../accessanalyzer/analyzer_archive_rules.go | 7 +- .../accessanalyzer/analyzer_findings.go | 7 +- .../services/accessanalyzer/analyzers.go | 7 +- .../accessanalyzer/analyzers_mock_test.go | 7 +- resources/services/acm/certificates.go | 9 +- .../services/acm/certificates_mock_test.go | 7 +- .../acmpca/certificate_authorities.go | 9 +- .../certificate_authorities_mock_test.go | 7 +- .../services/amp/rule_groups_namespaces.go | 7 +- .../amp/rule_groups_namespaces_mock_test.go | 5 +- resources/services/amp/workspaces.go | 9 +- .../services/amp/workspaces_mock_test.go | 7 +- resources/services/amplify/apps.go | 7 +- resources/services/amplify/apps_mock_test.go | 7 +- resources/services/apigateway/api_keys.go | 8 +- .../services/apigateway/api_keys_mock_test.go | 7 +- .../apigateway/client_certificates.go | 8 +- .../client_certificates_mock_test.go | 7 +- .../domain_name_base_path_mappings.go | 7 +- resources/services/apigateway/domain_names.go | 8 +- .../apigateway/domain_names_mock_test.go | 7 +- .../apigateway/rest_api_authorizers.go | 7 +- .../apigateway/rest_api_deployments.go | 7 +- .../rest_api_documentation_parts.go | 8 +- .../rest_api_documentation_versions.go | 8 +- .../apigateway/rest_api_gateway_responses.go | 9 +- .../services/apigateway/rest_api_models.go | 8 +- .../apigateway/rest_api_request_validators.go | 8 +- .../rest_api_resource_method_integration.go | 8 +- .../apigateway/rest_api_resource_methods.go | 7 +- .../services/apigateway/rest_api_resources.go | 8 +- .../services/apigateway/rest_api_stages.go | 8 +- resources/services/apigateway/rest_apis.go | 8 +- .../apigateway/rest_apis_mock_test.go | 7 +- .../services/apigateway/usage_plan_keys.go | 7 +- resources/services/apigateway/usage_plans.go | 10 +- .../apigateway/usage_plans_mock_test.go | 7 +- resources/services/apigateway/vpc_links.go | 8 +- .../apigateway/vpc_links_mock_test.go | 7 +- .../services/apigatewayv2/api_authorizers.go | 7 +- .../services/apigatewayv2/api_deployments.go | 8 +- .../apigatewayv2/api_integration_responses.go | 7 +- .../services/apigatewayv2/api_integrations.go | 7 +- resources/services/apigatewayv2/api_models.go | 8 +- .../apigatewayv2/api_route_responses.go | 7 +- resources/services/apigatewayv2/api_routes.go | 7 +- resources/services/apigatewayv2/api_stages.go | 7 +- resources/services/apigatewayv2/apis.go | 7 +- .../services/apigatewayv2/apis_mock_test.go | 7 +- .../domain_name_rest_api_mappings.go | 7 +- .../services/apigatewayv2/domain_names.go | 7 +- .../apigatewayv2/domain_names_mock_test.go | 7 +- resources/services/apigatewayv2/vpc_links.go | 7 +- .../apigatewayv2/vpc_links_mock_test.go | 7 +- resources/services/appconfig/applications.go | 7 +- .../appconfig/applications_mock_test.go | 7 +- .../appconfig/configuration_profiles.go | 7 +- .../appconfig/deployment_strategies.go | 7 +- .../deployment_strategies_mock_test.go | 7 +- resources/services/appconfig/environments.go | 7 +- .../appconfig/hosted_config_versions.go | 7 +- resources/services/appflow/flows.go | 7 +- resources/services/appflow/flows_mock_test.go | 7 +- .../applicationautoscaling/policies.go | 7 +- .../policies_mock_test.go | 7 +- .../scalable_targets.go | 7 +- .../scalable_targets_mock_test.go | 7 +- .../scheduled_actions.go | 7 +- .../scheduled_actions_mock_test.go | 7 +- resources/services/appmesh/meshes.go | 9 +- .../services/appmesh/meshes_mock_test.go | 7 +- .../services/appmesh/virtual_gateways.go | 7 +- resources/services/appmesh/virtual_nodes.go | 7 +- resources/services/appmesh/virtual_routers.go | 7 +- .../services/appmesh/virtual_services.go | 7 +- .../auto_scaling_configuration_mock_test.go | 7 +- .../apprunner/auto_scaling_configurations.go | 10 +- resources/services/apprunner/connections.go | 9 +- .../apprunner/connections_mock_test.go | 7 +- .../services/apprunner/custom_domains.go | 7 +- .../observability_configuration_mock_test.go | 7 +- .../apprunner/observability_configurations.go | 9 +- resources/services/apprunner/operations.go | 7 +- resources/services/apprunner/services.go | 9 +- .../services/apprunner/services_mock_test.go | 7 +- resources/services/apprunner/tag_fetch.go | 5 +- .../apprunner/vpc_connector_mock_test.go | 7 +- .../services/apprunner/vpc_connectors.go | 9 +- .../vpc_ingress_connection_mock_test.go | 7 +- .../apprunner/vpc_ingress_connections.go | 9 +- resources/services/appstream/app_blocks.go | 7 +- .../appstream/app_blocks_mock_test.go | 8 +- .../application_fleet_associations.go | 7 +- resources/services/appstream/applications.go | 7 +- .../appstream/applications_mock_test.go | 7 +- resources/services/appstream/fleets.go | 7 +- .../services/appstream/fleets_mock_test.go | 8 +- .../services/appstream/image_builders.go | 7 +- .../appstream/image_builders_mock_test.go | 8 +- resources/services/appstream/images.go | 7 +- .../services/appstream/images_mock_test.go | 8 +- .../services/appstream/stack_entitlements.go | 7 +- .../appstream/stack_user_associations.go | 7 +- resources/services/appstream/stacks.go | 7 +- .../services/appstream/stacks_mock_test.go | 7 +- resources/services/appstream/users.go | 7 +- .../services/appstream/users_mock_test.go | 7 +- resources/services/appsync/graphql_apis.go | 7 +- .../appsync/graphql_apis_mock_test.go | 7 +- .../athena/data_catalog_database_tables.go | 7 +- .../services/athena/data_catalog_databases.go | 7 +- resources/services/athena/data_catalogs.go | 9 +- .../athena/data_catalogs_mock_test.go | 7 +- .../athena/work_group_named_queries.go | 7 +- .../athena/work_group_prepared_statements.go | 7 +- .../athena/work_group_query_executions.go | 7 +- resources/services/athena/work_groups.go | 9 +- .../services/athena/work_groups_mock_test.go | 7 +- resources/services/auditmanager/assesments.go | 7 +- .../auditmanager/assesments_mock_test.go | 7 +- .../autoscaling/group_lifecycle_hooks.go | 9 +- .../autoscaling/group_scaling_policies.go | 9 +- resources/services/autoscaling/groups.go | 12 +- .../services/autoscaling/groups_mock_test.go | 7 +- .../launch_configuration_mock_test.go | 7 +- .../autoscaling/launch_configurations.go | 7 +- .../services/autoscaling/scheduled_actions.go | 7 +- .../scheduled_actions_mock_test.go | 7 +- resources/services/backup/plan_selections.go | 7 +- resources/services/backup/plans.go | 9 +- resources/services/backup/plans_mock_test.go | 7 +- .../services/backup/protected_resources.go | 7 +- .../backup/protected_resources_mock_test.go | 7 +- resources/services/backup/report_plans.go | 9 +- .../services/backup/report_plans_mock_test.go | 7 +- .../services/backup/vault_recovery_points.go | 9 +- resources/services/backup/vaults.go | 9 +- resources/services/backup/vaults_mock_test.go | 7 +- .../services/batch/compute_environments.go | 9 +- .../batch/compute_environments_mock_test.go | 7 +- resources/services/batch/job_definitions.go | 9 +- .../batch/job_definitions_mock_test.go | 7 +- resources/services/batch/job_queues.go | 9 +- .../services/batch/job_queues_mock_test.go | 7 +- resources/services/batch/jobs.go | 9 +- resources/services/batch/jobs_mock_test.go | 7 +- .../cloudformation/models/stacksets.go | 3 + .../stack_instance_resource_drifts.go | 10 +- .../cloudformation/stack_instances.go | 10 +- .../cloudformation/stack_resources.go | 7 +- .../cloudformation/stack_templates.go | 10 +- resources/services/cloudformation/stacks.go | 9 +- .../cloudformation/stacks_mock_test.go | 7 +- .../cloudformation/stackset_mock_test.go | 7 +- .../stackset_operation_results.go | 10 +- .../cloudformation/stackset_operations.go | 10 +- .../services/cloudformation/stacksets.go | 11 +- .../cloudformation/template_summaries.go | 9 +- .../services/cloudfront/cache_policies.go | 7 +- .../cloudfront/cache_policies_mock_test.go | 7 +- .../services/cloudfront/distributions.go | 9 +- .../cloudfront/distributions_mock_test.go | 7 +- resources/services/cloudfront/functions.go | 8 +- .../cloudfront/functions_mock_test.go | 7 +- resources/services/cloudhsmv2/backups.go | 9 +- .../services/cloudhsmv2/backups_mock_test.go | 7 +- resources/services/cloudhsmv2/clusters.go | 9 +- .../services/cloudhsmv2/clusters_mock_test.go | 7 +- resources/services/cloudtrail/channels.go | 8 +- .../services/cloudtrail/channels_mock_test.go | 7 +- .../cloudtrail/trail_event_selectors.go | 7 +- resources/services/cloudtrail/trails.go | 11 +- .../services/cloudtrail/trails_mock_test.go | 7 +- resources/services/cloudwatch/alarms.go | 10 +- .../services/cloudwatch/alarms_mock_test.go | 7 +- .../data_protection_policies.go | 8 +- .../services/cloudwatchlogs/log_groups.go | 9 +- .../cloudwatchlogs/log_groups_mock_test.go | 7 +- .../cloudwatchlogs/subscription_filters.go | 8 +- resources/services/codeartifact/domains.go | 9 +- .../codeartifact/domains_mock_test.go | 7 +- resources/services/codeartifact/helpers.go | 6 +- .../services/codeartifact/repositories.go | 9 +- .../codeartifact/repositories_mock_test.go | 7 +- resources/services/codebuild/builds.go | 7 +- resources/services/codebuild/projects.go | 9 +- .../services/codebuild/projects_mock_test.go | 7 +- .../services/codebuild/source_credentials.go | 7 +- .../codebuild/source_credentials_mock_test.go | 7 +- resources/services/codecommit/repositories.go | 9 +- .../codecommit/repositories_mock_test.go | 7 +- resources/services/codepipeline/pipelines.go | 9 +- .../codepipeline/pipelines_mock_test.go | 7 +- resources/services/codepipeline/webhooks.go | 9 +- .../codepipeline/webhooks_mock_test.go | 7 +- resources/services/cognito/identity_pools.go | 9 +- .../cognito/identity_pools_mock_test.go | 7 +- .../cognito/user_pool_identity_providers.go | 7 +- resources/services/cognito/user_pools.go | 9 +- .../services/cognito/user_pools_mock_test.go | 7 +- .../config/config_rule_compliance_details.go | 7 +- ...onfig_rule_compliance_details_mock_test.go | 7 +- .../config/config_rule_compliances.go | 7 +- resources/services/config/config_rules.go | 7 +- .../services/config/config_rules_mock_test.go | 7 +- .../config/configuration_aggregators.go | 7 +- .../configuration_aggregators_mock_test.go | 7 +- .../config/configuration_recorders.go | 9 +- .../configuration_recorders_mock_test.go | 7 +- .../config/conformance_pack_mock_test.go | 7 +- .../conformance_pack_rule_compliances.go | 9 +- .../services/config/conformance_packs.go | 7 +- .../config/remediation_configurations.go | 7 +- .../remediation_configurations_mock_test.go | 7 +- resources/services/dax/clusters.go | 10 +- resources/services/dax/clusters_mock_test.go | 7 +- resources/services/detective/graphs.go | 10 +- .../services/detective/graphs_mock_test.go | 7 +- resources/services/detective/members.go | 7 +- .../services/directconnect/connections.go | 9 +- .../directconnect/connections_mock_test.go | 7 +- .../directconnect/gateway_associations.go | 7 +- .../directconnect/gateway_attachments.go | 7 +- .../directconnect/gateway_mock_test.go | 7 +- resources/services/directconnect/gateways.go | 7 +- resources/services/directconnect/lags.go | 9 +- .../services/directconnect/lags_mock_test.go | 7 +- .../directconnect/virtual_interfaces.go | 9 +- .../virtual_interfaces_mock_test.go | 7 +- .../services/dms/replication_instances.go | 9 +- .../dms/replication_instances_mock_test.go | 7 +- resources/services/dms/tags.go | 3 +- resources/services/dms/tags_test.go | 3 +- resources/services/docdb/certificates.go | 7 +- .../services/docdb/certificates_mock_test.go | 7 +- .../docdb/cluster_parameter_groups.go | 9 +- .../cluster_parameter_groups_mock_test.go | 7 +- resources/services/docdb/cluster_snapshots.go | 9 +- resources/services/docdb/clusters.go | 9 +- .../services/docdb/clusters_mock_test.go | 7 +- .../services/docdb/event_subscriptions.go | 7 +- .../docdb/event_subscriptions_mock_test.go | 7 +- resources/services/docdb/global_clusters.go | 7 +- resources/services/docdb/helpers.go | 5 +- resources/services/docdb/instances.go | 9 +- resources/services/docdb/subnet_groups.go | 9 +- .../services/docdb/subnet_groups_mock_test.go | 7 +- resources/services/dynamodb/backups.go | 7 +- .../services/dynamodb/backups_mock_test.go | 7 +- .../services/dynamodb/export_mock_test.go | 7 +- resources/services/dynamodb/exports.go | 7 +- resources/services/dynamodb/global_tables.go | 7 +- .../dynamodb/global_tables_mock_test.go | 7 +- .../dynamodb/table_continuous_backups.go | 7 +- .../dynamodb/table_replica_auto_scalings.go | 7 +- resources/services/dynamodb/tables.go | 9 +- .../services/dynamodb/tables_mock_test.go | 7 +- resources/services/dynamodbstreams/streams.go | 7 +- .../dynamodbstreams/streams_mock_test.go | 7 +- .../services/ec2/capacity_reservations.go | 9 +- .../ec2/capacity_reservations_mock_test.go | 7 +- resources/services/ec2/customer_gateways.go | 9 +- .../ec2/customer_gateways_mock_test.go | 7 +- .../services/ec2/ebs_snapshot_attributes.go | 7 +- resources/services/ec2/ebs_snapshots.go | 9 +- .../services/ec2/ebs_snapshots_mock_test.go | 7 +- resources/services/ec2/ebs_volumes.go | 9 +- .../services/ec2/ebs_volumes_mock_test.go | 7 +- .../ec2/egress_only_internet_gateways.go | 9 +- ...egress_only_internet_gateways_mock_test.go | 7 +- resources/services/ec2/eips.go | 9 +- resources/services/ec2/flow_logs.go | 9 +- resources/services/ec2/flow_logs_mock_test.go | 8 +- resources/services/ec2/hosts.go | 9 +- resources/services/ec2/hosts_mock_test.go | 7 +- resources/services/ec2/image_attributes.go | 7 +- resources/services/ec2/image_last_launched.go | 5 +- resources/services/ec2/images.go | 9 +- resources/services/ec2/images_mock_test.go | 7 +- resources/services/ec2/instance_connect.go | 9 +- .../ec2/instance_connect_mock_test.go | 7 +- resources/services/ec2/instances.go | 9 +- resources/services/ec2/instances_mock_test.go | 7 +- resources/services/ec2/internet_gateways.go | 9 +- .../ec2/internet_gateways_mock_test.go | 7 +- resources/services/ec2/key_pairs.go | 9 +- resources/services/ec2/key_pairs_mock_test.go | 7 +- .../services/ec2/launch_template_versions.go | 7 +- .../ec2/launch_template_versions_mock_test.go | 7 +- resources/services/ec2/launch_templates.go | 9 +- .../ec2/launch_templates_mock_test.go | 7 +- .../services/ec2/managed_prefix_lists.go | 10 +- .../ec2/managed_prefix_lists_mock_test.go | 7 +- resources/services/ec2/nat_gateways.go | 9 +- .../services/ec2/nat_gateways_mock_test.go | 7 +- resources/services/ec2/network_acls.go | 9 +- .../services/ec2/network_acls_mock_test.go | 7 +- resources/services/ec2/network_interfaces.go | 9 +- .../ec2/network_interfaces_mock_test.go | 7 +- .../services/ec2/regional_config_mock_test.go | 5 +- resources/services/ec2/regional_configs.go | 9 +- resources/services/ec2/regions.go | 7 +- resources/services/ec2/regions_mock_test.go | 7 +- resources/services/ec2/reserved_instances.go | 9 +- .../ec2/reserved_instances_mock_test.go | 7 +- resources/services/ec2/route_tables.go | 9 +- .../services/ec2/route_tables_mock_test.go | 7 +- resources/services/ec2/security_groups.go | 9 +- .../services/ec2/security_groups_mock_test.go | 7 +- resources/services/ec2/subnets.go | 9 +- resources/services/ec2/subnets_mock_test.go | 7 +- .../ec2/transit_gateway_attachments.go | 7 +- .../ec2/transit_gateway_multicast_domains.go | 7 +- .../transit_gateway_peering_attachments.go | 7 +- .../ec2/transit_gateway_route_tables.go | 7 +- .../ec2/transit_gateway_vpc_attachments.go | 7 +- resources/services/ec2/transit_gateways.go | 7 +- .../ec2/transit_gateways_mock_test.go | 7 +- .../ec2/vpc_endpoint_service_permissions.go | 10 +- .../services/ec2/vpc_endpoint_services.go | 10 +- .../ec2/vpc_endpoint_services_mock_test.go | 7 +- resources/services/ec2/vpc_endpoints.go | 10 +- .../services/ec2/vpc_endpoints_mock_test.go | 7 +- .../services/ec2/vpc_peering_connections.go | 9 +- resources/services/ec2/vpcs.go | 9 +- resources/services/ec2/vpcs_mock_test.go | 8 +- .../ec2/vpcs_peering_connections_mock_test.go | 7 +- resources/services/ec2/vpn_connections.go | 9 +- .../services/ec2/vpn_connections_mock_test.go | 7 +- resources/services/ec2/vpn_gateways.go | 9 +- .../services/ec2/vpn_gateways_mock_test.go | 7 +- resources/services/ecr/lifecycle_policy.go | 10 +- resources/services/ecr/repositories.go | 10 +- .../services/ecr/repositories_mock_test.go | 7 +- .../ecr/repository_image_scan_findings.go | 8 +- resources/services/ecr/repository_images.go | 8 +- resources/services/ecr/repository_policy.go | 10 +- resources/services/ecrpublic/repositories.go | 9 +- .../ecrpublic/repositories_mock_test.go | 7 +- .../services/ecrpublic/repository_images.go | 7 +- .../ecs/cluster_container_instances.go | 9 +- resources/services/ecs/cluster_mock_test.go | 7 +- resources/services/ecs/cluster_services.go | 9 +- resources/services/ecs/cluster_task_sets.go | 10 +- resources/services/ecs/cluster_tasks.go | 9 +- resources/services/ecs/clusters.go | 9 +- resources/services/ecs/task_definitions.go | 11 +- .../ecs/task_definitions_mock_test.go | 7 +- resources/services/efs/accesspoints.go | 9 +- .../services/efs/accesspoints_mock_test.go | 7 +- resources/services/efs/filesystems.go | 9 +- .../services/efs/filesystems_mock_test.go | 7 +- resources/services/eks/add_ons.go | 7 +- resources/services/eks/clusters.go | 7 +- resources/services/eks/clusters_mock_test.go | 7 +- resources/services/eks/fargate_profiles.go | 7 +- .../services/eks/identity_provider_configs.go | 7 +- resources/services/eks/node_groups.go | 7 +- resources/services/elasticache/clusters.go | 9 +- .../elasticache/clusters_mock_test.go | 7 +- .../elasticache/global_replication_groups.go | 7 +- .../global_replication_groups_mock_test.go | 7 +- .../services/elasticache/parameter_groups.go | 7 +- .../elasticache/parameter_groups_mock_test.go | 7 +- .../elasticache/replication_groups.go | 9 +- .../replication_groups_mock_test.go | 7 +- .../elasticache/reserved_cache_nodes.go | 7 +- .../reserved_cache_nodes_mock_test.go | 7 +- .../services/elasticache/service_updates.go | 7 +- .../elasticache/service_updates_mock_test.go | 7 +- resources/services/elasticache/snapshots.go | 7 +- .../elasticache/snapshots_mock_test.go | 7 +- .../services/elasticache/subnet_groups.go | 7 +- .../elasticache/subnet_groups_mock_test.go | 7 +- resources/services/elasticache/user_groups.go | 7 +- .../elasticache/user_groups_mock_test.go | 7 +- resources/services/elasticache/users.go | 7 +- .../services/elasticache/users_mock_test.go | 7 +- .../elasticbeanstalk/application_versions.go | 7 +- .../application_versions_mock_test.go | 7 +- .../services/elasticbeanstalk/applications.go | 9 +- .../applications_mock_test.go | 7 +- .../elasticbeanstalk/configuration_options.go | 9 +- .../configuration_settings.go | 9 +- .../services/elasticbeanstalk/environments.go | 11 +- .../envrionments_mock_test.go | 7 +- resources/services/elasticsearch/domains.go | 9 +- .../elasticsearch/domains_mock_test.go | 7 +- .../elastictranscoder/pipeline_jobs.go | 7 +- .../services/elastictranscoder/pipelines.go | 7 +- .../elastictranscoder/pipelines_mock_test.go | 8 +- .../services/elastictranscoder/presets.go | 7 +- .../elastictranscoder/presets_mock_test.go | 8 +- .../services/elbv1/load_balancer_policies.go | 12 +- resources/services/elbv1/load_balancers.go | 9 +- .../elbv1/load_balancers_mock_test.go | 7 +- .../services/elbv2/listener_certificates.go | 7 +- resources/services/elbv2/listener_rules.go | 7 +- resources/services/elbv2/listeners.go | 9 +- .../elbv2/load_balancer_attributes.go | 7 +- .../services/elbv2/load_balancer_web_acls.go | 4 +- resources/services/elbv2/load_balancers.go | 9 +- .../elbv2/load_balancers_mock_test.go | 7 +- ...target_group_target_health_descriptions.go | 7 +- resources/services/elbv2/target_groups.go | 9 +- .../services/elbv2/target_groups_mock_test.go | 7 +- .../services/emr/cluster_instance_fleets.go | 7 +- .../services/emr/cluster_instance_groups.go | 7 +- resources/services/emr/cluster_instances.go | 7 +- resources/services/emr/clusters.go | 9 +- resources/services/emr/clusters_mock_test.go | 7 +- resources/services/emr/notebook_executions.go | 9 +- resources/services/emr/steps.go | 7 +- .../services/emr/studio_session_mapping.go | 7 +- resources/services/emr/studios.go | 9 +- resources/services/emr/studios_test.go | 10 +- .../services/eventbridge/api_destinations.go | 7 +- .../eventbridge/api_destinations_mock_test.go | 8 +- resources/services/eventbridge/archives.go | 7 +- .../eventbridge/archives_mock_test.go | 8 +- resources/services/eventbridge/connections.go | 7 +- .../eventbridge/connections_mock_test.go | 8 +- resources/services/eventbridge/endpoints.go | 7 +- .../eventbridge/endpoints_mock_test.go | 8 +- .../services/eventbridge/event_bus_rules.go | 9 +- .../services/eventbridge/event_bus_targets.go | 7 +- resources/services/eventbridge/event_buses.go | 9 +- .../eventbridge/event_buses_mock_test.go | 7 +- .../services/eventbridge/event_sources.go | 7 +- .../eventbridge/event_sources_mock_test.go | 8 +- resources/services/eventbridge/replays.go | 7 +- .../services/eventbridge/replays_mock_test.go | 8 +- .../services/firehose/delivery_streams.go | 9 +- .../firehose/delivery_streams_mock_test.go | 7 +- .../services/frauddetector/batch_imports.go | 7 +- .../frauddetector/batch_imports_mock_test.go | 7 +- .../frauddetector/batch_predictions.go | 7 +- .../batch_predictions_mock_test.go | 7 +- resources/services/frauddetector/detectors.go | 9 +- .../frauddetector/detectors_mock_test.go | 7 +- .../services/frauddetector/entity_types.go | 9 +- .../frauddetector/entity_types_mock_test.go | 7 +- .../services/frauddetector/event_types.go | 9 +- .../frauddetector/event_types_mock_test.go | 7 +- .../services/frauddetector/external_models.go | 7 +- .../external_models_mock_test.go | 7 +- resources/services/frauddetector/labels.go | 6 +- .../frauddetector/labels_mock_test.go | 7 +- .../services/frauddetector/model_versions.go | 7 +- .../frauddetector/model_versions_mock_test.go | 5 +- resources/services/frauddetector/models.go | 7 +- .../frauddetector/models_mock_test.go | 7 +- resources/services/frauddetector/outcomes.go | 9 +- .../frauddetector/outcomes_mock_test.go | 7 +- resources/services/frauddetector/rules.go | 7 +- .../services/frauddetector/rules_mock_test.go | 5 +- resources/services/frauddetector/tag_fetch.go | 5 +- .../services/frauddetector/tags_mock_test.go | 5 +- resources/services/frauddetector/variables.go | 9 +- .../frauddetector/variables_mock_test.go | 7 +- .../fsx/data_repository_associations.go | 9 +- .../fsx/data_repository_associations_test.go | 7 +- .../services/fsx/data_repository_tasks.go | 9 +- .../fsx/data_repository_tasks_test.go | 7 +- resources/services/fsx/file_caches.go | 9 +- resources/services/fsx/file_caches_test.go | 7 +- resources/services/fsx/file_systems.go | 9 +- resources/services/fsx/file_systems_test.go | 7 +- resources/services/fsx/snapshots.go | 9 +- resources/services/fsx/snapshots_test.go | 7 +- .../services/fsx/storage_virtual_machines.go | 9 +- .../fsx/storage_virtual_machines_test.go | 7 +- resources/services/fsx/volumes.go | 9 +- resources/services/fsx/volumes_test.go | 7 +- .../services/glacier/vault_access_policies.go | 9 +- .../services/glacier/vault_lock_policies.go | 9 +- .../services/glacier/vault_notifications.go | 7 +- resources/services/glacier/vaults.go | 9 +- .../services/glacier/vaults_mock_test.go | 7 +- resources/services/glue/connections.go | 8 +- resources/services/glue/connections_test.go | 7 +- resources/services/glue/crawlers.go | 7 +- resources/services/glue/crawlers_test.go | 7 +- .../services/glue/database_table_indexes.go | 7 +- resources/services/glue/database_tables.go | 7 +- resources/services/glue/databases.go | 8 +- .../services/glue/databases_mock_test.go | 7 +- resources/services/glue/dev_endpoints.go | 8 +- resources/services/glue/dev_endpoints_test.go | 7 +- resources/services/glue/job_runs.go | 7 +- resources/services/glue/jobs.go | 7 +- resources/services/glue/jobs_test.go | 7 +- .../services/glue/ml_transform_task_runs.go | 7 +- resources/services/glue/ml_transforms.go | 10 +- resources/services/glue/ml_transforms_test.go | 7 +- resources/services/glue/registries.go | 7 +- resources/services/glue/registries_test.go | 7 +- .../services/glue/registry_schema_versions.go | 9 +- resources/services/glue/registry_schemas.go | 7 +- .../services/glue/security_configurations.go | 7 +- .../glue/security_configurations_test.go | 7 +- resources/services/glue/tags.go | 7 +- resources/services/glue/triggers.go | 7 +- resources/services/glue/triggers_test.go | 7 +- resources/services/glue/workflows.go | 7 +- resources/services/glue/workflows_test.go | 7 +- .../services/guardduty/detector_filters.go | 9 +- .../services/guardduty/detector_findings.go | 9 +- .../services/guardduty/detector_ipsets.go | 9 +- .../services/guardduty/detector_members.go | 9 +- .../detector_publishing_destinations.go | 9 +- .../guardduty/detector_threat_intel_sets.go | 9 +- resources/services/guardduty/detectors.go | 9 +- .../services/guardduty/detectors_mock_test.go | 7 +- resources/services/iam/credential_reports.go | 10 +- .../iam/credential_reports_mock_test.go | 7 +- .../services/iam/group_attached_policies.go | 10 +- resources/services/iam/group_policies.go | 12 +- resources/services/iam/groups.go | 10 +- resources/services/iam/groups_mock_test.go | 7 +- resources/services/iam/instance_profiles.go | 9 +- .../iam/instance_profiles_mock_test.go | 7 +- resources/services/iam/mfa_devices.go | 7 +- .../iam/openid_connect_identity_providers.go | 11 +- ...id_connect_identity_providers_mock_test.go | 7 +- resources/services/iam/password_policies.go | 9 +- .../iam/password_policies_mock_test.go | 7 +- resources/services/iam/policies.go | 9 +- resources/services/iam/policies_mock_test.go | 7 +- resources/services/iam/policy_versions.go | 10 +- .../services/iam/role_attached_policies.go | 7 +- resources/services/iam/role_policies.go | 9 +- resources/services/iam/roles.go | 9 +- resources/services/iam/roles_mock_test.go | 7 +- .../services/iam/saml_identity_providers.go | 9 +- .../iam/saml_identity_providers_mock_test.go | 7 +- resources/services/iam/server_certificates.go | 7 +- .../iam/server_certificates_mock_test.go | 7 +- .../services/iam/signing_certificates.go | 7 +- resources/services/iam/ssh_public_keys.go | 8 +- resources/services/iam/user_access_keys.go | 9 +- .../services/iam/user_attached_policies.go | 7 +- resources/services/iam/user_groups.go | 8 +- resources/services/iam/user_policies.go | 9 +- resources/services/iam/users.go | 9 +- resources/services/iam/users_mock_test.go | 7 +- .../identitystore/group_memberships.go | 7 +- resources/services/identitystore/groups.go | 7 +- .../identitystore/groups_mock_test.go | 7 +- .../services/identitystore/instance_fetch.go | 5 +- resources/services/identitystore/users.go | 7 +- .../services/identitystore/users_mock_test.go | 7 +- resources/services/inspector/findings.go | 9 +- .../services/inspector/findings_mock_test.go | 7 +- resources/services/inspector2/findings.go | 7 +- .../services/inspector2/findings_mock_test.go | 7 +- resources/services/iot/billing_groups.go | 10 +- .../services/iot/billing_groups_mock_test.go | 7 +- resources/services/iot/ca_certificates.go | 7 +- .../services/iot/ca_certificates_mock_test.go | 7 +- resources/services/iot/certificates.go | 8 +- .../services/iot/certificates_mock_test.go | 7 +- resources/services/iot/jobs.go | 9 +- resources/services/iot/jobs_mock_test.go | 7 +- resources/services/iot/policies.go | 9 +- resources/services/iot/policies_mock_test.go | 7 +- resources/services/iot/security_profiles.go | 10 +- .../iot/security_profiles_mock_test.go | 7 +- resources/services/iot/streams.go | 7 +- resources/services/iot/streams_mock_test.go | 7 +- resources/services/iot/tags.go | 5 +- resources/services/iot/thing_groups.go | 11 +- .../services/iot/thing_groups_mock_test.go | 7 +- resources/services/iot/thing_types.go | 10 +- .../services/iot/thing_types_mock_test.go | 7 +- resources/services/iot/things.go | 8 +- resources/services/iot/things_mock_test.go | 7 +- resources/services/iot/topic_rules.go | 9 +- .../services/iot/topic_rules_mock_test.go | 7 +- .../services/kafka/cluster_operations.go | 9 +- .../kafka/cluster_operations_mock_test.go | 5 +- resources/services/kafka/clusters.go | 7 +- .../services/kafka/clusters_mock_test.go | 8 +- resources/services/kafka/configurations.go | 7 +- .../kafka/configurations_mock_test.go | 8 +- resources/services/kafka/helpers.go | 5 +- resources/services/kafka/nodes.go | 7 +- resources/services/kafka/nodes_mock_test.go | 5 +- resources/services/kinesis/streams.go | 9 +- .../services/kinesis/streams_mock_test.go | 7 +- resources/services/kms/aliases.go | 7 +- resources/services/kms/aliases_mock_test.go | 7 +- resources/services/kms/key_grants.go | 7 +- resources/services/kms/key_policies.go | 9 +- resources/services/kms/keys.go | 9 +- resources/services/kms/keys_mock_test.go | 7 +- resources/services/lambda/function_aliases.go | 7 +- .../lambda/function_concurrency_configs.go | 7 +- .../lambda/function_event_invoke_configs.go | 8 +- .../lambda/function_event_source_mappings.go | 9 +- .../services/lambda/function_url_configs.go | 7 +- .../services/lambda/function_versions.go | 7 +- resources/services/lambda/functions.go | 9 +- .../services/lambda/functions_mock_test.go | 7 +- .../services/lambda/layer_version_policies.go | 7 +- resources/services/lambda/layer_versions.go | 7 +- resources/services/lambda/layers.go | 9 +- resources/services/lambda/layers_mock_test.go | 7 +- resources/services/lightsail/alarms.go | 7 +- .../services/lightsail/alarms_mock_test.go | 7 +- .../services/lightsail/bucket_access_keys.go | 7 +- resources/services/lightsail/buckets.go | 9 +- .../services/lightsail/buckets_mock_test.go | 7 +- resources/services/lightsail/certificates.go | 9 +- .../lightsail/certificates_mock_test.go | 7 +- .../container_service_deployments.go | 7 +- .../lightsail/container_service_images.go | 7 +- .../services/lightsail/container_services.go | 9 +- .../lightsail/container_services_mock_test.go | 7 +- .../services/lightsail/database_events.go | 7 +- .../services/lightsail/database_log_events.go | 9 +- .../services/lightsail/database_parameters.go | 7 +- .../services/lightsail/database_snapshots.go | 9 +- .../lightsail/database_snapshots_mock_test.go | 7 +- resources/services/lightsail/databases.go | 9 +- .../services/lightsail/databases_mock_test.go | 7 +- .../services/lightsail/disk_snapshots.go | 9 +- resources/services/lightsail/disks.go | 9 +- .../services/lightsail/disks_mock_test.go | 7 +- resources/services/lightsail/distributions.go | 11 +- .../lightsail/distributions_mock_test.go | 7 +- .../lightsail/instance_port_states.go | 7 +- .../services/lightsail/instance_snapshots.go | 9 +- .../lightsail/instance_snapshots_mock_test.go | 7 +- resources/services/lightsail/instances.go | 10 +- .../services/lightsail/instances_mock_test.go | 7 +- .../load_balancer_tls_certificates.go | 9 +- .../services/lightsail/load_balancers.go | 9 +- .../lightsail/load_balancers_mock_test.go | 7 +- resources/services/lightsail/static_ips.go | 7 +- .../lightsail/static_ips_mock_test.go | 7 +- .../mq/broker_configuration_revisions.go | 12 +- .../services/mq/broker_configurations.go | 7 +- resources/services/mq/broker_users.go | 7 +- resources/services/mq/brokers.go | 7 +- resources/services/mq/brokers_mock_test.go | 7 +- resources/services/mwaa/environments.go | 7 +- .../services/mwaa/environments_mock_test.go | 7 +- .../cluster_parameter_group_parameters.go | 7 +- .../neptune/cluster_parameter_groups.go | 9 +- .../cluster_parameter_groups_mock_test.go | 7 +- .../services/neptune/cluster_snapshots.go | 9 +- .../neptune/cluster_snapshots_mock_test.go | 7 +- resources/services/neptune/clusters.go | 9 +- .../db_parameter_group_db_parameters.go | 7 +- .../services/neptune/db_parameter_groups.go | 9 +- .../neptune/db_parameter_groups_mock_test.go | 7 +- .../services/neptune/event_subscriptions.go | 9 +- .../neptune/event_subscriptions_mock_test.go | 7 +- resources/services/neptune/global_clusters.go | 7 +- .../neptune/global_clusters_mock_test.go | 7 +- resources/services/neptune/instances.go | 9 +- resources/services/neptune/mock_test.go | 9 +- resources/services/neptune/subnet_groups.go | 9 +- .../networkfirewall/firewall_policies.go | 11 +- .../firewall_policies_mock_test.go | 7 +- .../services/networkfirewall/firewalls.go | 11 +- .../networkfirewall/firewalls_mock_test.go | 7 +- .../networkfirewall/models/networkfirewall.go | 1 + .../services/networkfirewall/rule_groups.go | 11 +- .../networkfirewall/rule_groups_mock_test.go | 7 +- .../tls_inspection_configurations.go | 11 +- ...tls_inspection_configurations_mock_test.go | 7 +- .../networkmanager/global_networks.go | 9 +- .../global_networks_mock_test.go | 7 +- resources/services/networkmanager/link.go | 9 +- resources/services/networkmanager/sites.go | 9 +- .../transit_gateway_registrations.go | 7 +- .../services/organizations/account_parents.go | 8 +- resources/services/organizations/accounts.go | 10 +- .../organizations/accounts_mock_test.go | 7 +- .../organizations/delegated_admins.go | 8 +- .../delegated_admins_mock_test.go | 7 +- .../organizations/delegated_services.go | 8 +- .../organizational_unit_parents.go | 8 +- .../organizations/organizational_units.go | 9 +- .../organizational_units_mock_test.go | 7 +- .../services/organizations/organizations.go | 8 +- .../organizations/organizations_mock_test.go | 7 +- resources/services/organizations/policies.go | 9 +- .../organizations/resource_policies.go | 7 +- .../resource_policies_mock_test.go | 7 +- resources/services/organizations/roots.go | 10 +- .../services/organizations/roots_mock_test.go | 7 +- .../qldb/ledger_journal_kinesis_streams.go | 7 +- .../qldb/ledger_journal_s3_exports.go | 7 +- resources/services/qldb/ledgers.go | 9 +- resources/services/qldb/ledgers_mock_test.go | 7 +- resources/services/quicksight/analyses.go | 7 +- .../services/quicksight/analyses_mock_test.go | 8 +- resources/services/quicksight/dashboards.go | 7 +- .../quicksight/dashboards_mock_test.go | 8 +- resources/services/quicksight/data_sets.go | 7 +- .../quicksight/data_sets_mock_test.go | 8 +- resources/services/quicksight/data_sources.go | 7 +- .../quicksight/data_sources_mock_test.go | 8 +- resources/services/quicksight/folders.go | 7 +- .../services/quicksight/folders_mock_test.go | 8 +- .../services/quicksight/group_members.go | 7 +- resources/services/quicksight/groups.go | 7 +- .../services/quicksight/groups_mock_test.go | 8 +- resources/services/quicksight/ingestions.go | 7 +- resources/services/quicksight/tag_fetch.go | 7 +- resources/services/quicksight/templates.go | 7 +- .../quicksight/templates_mock_test.go | 8 +- resources/services/quicksight/users.go | 7 +- .../services/quicksight/users_mock_test.go | 8 +- .../ram/resource_share_invitations.go | 7 +- .../resource_share_invitations_mock_test.go | 8 +- .../ram/resource_share_permissions.go | 9 +- resources/services/ram/resource_shares.go | 9 +- .../services/ram/resource_shares_mock_test.go | 7 +- resources/services/ram/resources.go | 7 +- resources/services/ram/resources_mock_test.go | 8 +- resources/services/rds/certificates.go | 7 +- resources/services/rds/cluster_backtracks.go | 7 +- .../rds/cluster_backtracks_mock_test.go | 5 +- .../rds/cluster_parameter_group_parameters.go | 7 +- .../services/rds/cluster_parameter_groups.go | 9 +- .../rds/cluster_parameter_groups_mock_test.go | 7 +- resources/services/rds/cluster_snapshots.go | 9 +- .../rds/cluster_snapshots_mock_test.go | 7 +- resources/services/rds/clusters.go | 9 +- .../rds/db_parameter_group_db_parameters.go | 7 +- resources/services/rds/db_parameter_groups.go | 9 +- .../rds/db_parameter_groups_mock_test.go | 7 +- resources/services/rds/db_proxies.go | 9 +- .../services/rds/db_proxies_mock_test.go | 7 +- resources/services/rds/db_security_groups.go | 9 +- .../rds/db_security_groups_mock_test.go | 7 +- resources/services/rds/db_snapshots.go | 11 +- .../services/rds/db_snapshots_mock_test.go | 7 +- resources/services/rds/event_subscriptions.go | 9 +- .../rds/event_subscriptions_mock_test.go | 7 +- resources/services/rds/fetch_rds_tags.go | 5 +- resources/services/rds/instances.go | 9 +- resources/services/rds/mock_test.go | 10 +- resources/services/rds/option_groups.go | 9 +- .../services/rds/option_groups_mock_test.go | 7 +- resources/services/rds/reserved_instances.go | 9 +- resources/services/rds/subnet_groups.go | 9 +- .../redshift/cluster_parameter_groups.go | 7 +- .../services/redshift/cluster_parameters.go | 7 +- resources/services/redshift/clusters.go | 9 +- .../services/redshift/clusters_mock_test.go | 7 +- resources/services/redshift/data_shares.go | 7 +- .../redshift/data_shares_mock_test.go | 7 +- .../services/redshift/endpoint_access.go | 7 +- .../redshift/endpoint_authorization.go | 7 +- .../services/redshift/event_subscriptions.go | 9 +- .../redshift/event_subscriptions_mock_test.go | 7 +- resources/services/redshift/snapshots.go | 9 +- resources/services/redshift/subnet_groups.go | 9 +- .../redshift/subnet_groups_mock_test.go | 7 +- .../resiliencehub/alarm_recommendations.go | 7 +- .../alarm_recommendations_mock_test.go | 5 +- .../services/resiliencehub/app_assessments.go | 7 +- .../app_assessments_mock_test.go | 5 +- .../app_component_compliances.go | 7 +- .../app_component_compliances_mock_test.go | 5 +- .../app_version_resource_mappings.go | 7 +- ...app_version_resource_mappings_mock_test.go | 5 +- .../resiliencehub/app_version_resources.go | 7 +- .../app_version_resources_mock_test.go | 5 +- .../services/resiliencehub/app_versions.go | 7 +- .../resiliencehub/app_versions_mock_test.go | 5 +- resources/services/resiliencehub/apps.go | 7 +- .../services/resiliencehub/apps_mock_test.go | 7 +- resources/services/resiliencehub/columns.go | 3 +- .../component_recommendations.go | 7 +- .../component_recommendations_mock_test.go | 5 +- .../resiliencehub/recommendation_templates.go | 7 +- .../recommendations_templates_mock_test.go | 5 +- .../resiliencehub/resiliency_policies.go | 7 +- .../resiliency_policies_mock_test.go | 7 +- .../resiliencehub/sop_recommendations.go | 7 +- .../sop_recommendations_mock_test.go | 5 +- .../suggested_resiliency_policies.go | 7 +- ...suggested_resiliency_policies_mock_test.go | 7 +- .../resiliencehub/test_recommendations.go | 7 +- .../test_recommendations_mock_test.go | 5 +- .../resourcegroups/resource_groups.go | 12 +- .../resource_groups_mock_test.go | 7 +- resources/services/route53/delegation_sets.go | 7 +- .../route53/delegations_sets_mock_test.go | 8 +- resources/services/route53/domains.go | 10 +- .../services/route53/domains_mock_test.go | 7 +- .../route53/health_check_mock_test.go | 7 +- resources/services/route53/health_checks.go | 9 +- .../hosted_zone_query_logging_configs.go | 9 +- .../hosted_zone_resource_record_sets.go | 9 +- .../hosted_zone_traffic_policy_instances.go | 9 +- resources/services/route53/hosted_zones.go | 9 +- .../route53/hosted_zones_mock_test.go | 7 +- .../services/route53/traffic_policies.go | 9 +- .../route53/traffic_policies_mock_test.go | 7 +- .../route53/traffic_policy_versions.go | 9 +- .../route53recoverycontrolconfig/clusters.go | 7 +- .../clusters_mock_test.go | 7 +- .../control_panel.go | 7 +- .../control_panel_mock_test.go | 7 +- .../routing_controls.go | 7 +- .../safety_rules.go | 7 +- .../route53recoveryreadiness/cells.go | 7 +- .../cells_mock_test.go | 7 +- .../readiness_checks.go | 7 +- .../readiness_checks_mock_test.go | 7 +- .../recovery_groups.go | 7 +- .../recovery_groups_mock_test.go | 7 +- .../route53recoveryreadiness/resource_sets.go | 7 +- .../resource_sets_mock_test.go | 7 +- .../route53resolver/firewall_domain_list.go | 8 +- .../firewall_domain_list_mock_test.go | 8 +- .../firewall_rule_group_associations.go | 7 +- ...ewall_rule_group_associations_mock_test.go | 8 +- .../firewall_rule_group_mock_test.go | 8 +- .../route53resolver/firewall_rule_groups.go | 7 +- .../route53resolver/resolver_endpoints.go | 7 +- .../resolver_endpoints_mock_test.go | 8 +- .../resolver_query_log_config_associations.go | 7 +- ...query_log_config_associations_mock_test.go | 8 +- .../resolver_query_log_configs.go | 7 +- .../resolver_query_log_configs_mock_test.go | 8 +- .../resolver_rule_associations.go | 7 +- .../resolver_rule_associations_mock_test.go | 8 +- .../route53resolver/resolver_rules.go | 7 +- .../resolver_rules_mock_test.go | 8 +- .../services/s3/access_grant_instances.go | 7 +- .../s3/access_grant_instances_mock_test.go | 7 +- resources/services/s3/access_grants.go | 7 +- .../services/s3/access_grants_mock_test.go | 7 +- resources/services/s3/access_points.go | 7 +- .../services/s3/access_points_mock_test.go | 7 +- resources/services/s3/bucket_cors_rules.go | 13 +- .../services/s3/bucket_encryption_rules.go | 12 +- resources/services/s3/bucket_grants.go | 13 +- resources/services/s3/bucket_lifecycles.go | 12 +- resources/services/s3/bucket_logging.go | 11 +- .../s3/bucket_notification_configurations.go | 12 +- .../s3/bucket_object_lock_configurations.go | 12 +- .../services/s3/bucket_ownership_controls.go | 11 +- resources/services/s3/bucket_policies.go | 13 +- .../services/s3/bucket_public_access_block.go | 11 +- resources/services/s3/bucket_replications.go | 11 +- resources/services/s3/bucket_versioning.go | 11 +- resources/services/s3/bucket_websites.go | 13 +- resources/services/s3/buckets.go | 10 +- resources/services/s3/buckets_mock_test.go | 7 +- .../services/s3/multi_region_access_points.go | 7 +- .../multi_region_access_points_mock_test.go | 7 +- resources/services/sagemaker/apps.go | 10 +- .../services/sagemaker/apps_mock_test.go | 7 +- .../sagemaker/endpoint_configurations.go | 9 +- .../endpoint_configurations_mock_test.go | 7 +- resources/services/sagemaker/models.go | 9 +- .../services/sagemaker/models_mock_test.go | 7 +- .../services/sagemaker/notebook_instances.go | 9 +- .../sagemaker/notebook_instances_mock_test.go | 7 +- resources/services/sagemaker/training_jobs.go | 9 +- .../sagemaker/training_jobs_mock_test.go | 7 +- .../services/savingsplans/savingsplans.go | 7 +- .../savingsplans/savingsplans_mock_test.go | 7 +- .../services/scheduler/schedule_groups.go | 9 +- .../scheduler/schedule_groups_mock_test.go | 8 +- resources/services/scheduler/schedules.go | 9 +- .../services/scheduler/schedules_mock_test.go | 8 +- resources/services/secretsmanager/secrets.go | 9 +- .../secretsmanager/secrets_mock_test.go | 7 +- .../secretsmanager/secrets_version.go | 7 +- .../services/securityhub/enabled_standards.go | 7 +- .../enabled_standards_mock_test.go | 7 +- resources/services/securityhub/hubs.go | 9 +- .../services/securityhub/hubs_mock_test.go | 7 +- .../services/servicecatalog/launch_paths.go | 10 +- .../services/servicecatalog/portfolios.go | 9 +- .../servicecatalog/portfolios_mock_test.go | 7 +- resources/services/servicecatalog/products.go | 9 +- .../servicecatalog/products_mock_test.go | 7 +- .../servicecatalog/provisioned_products.go | 9 +- .../provisioned_products_mock_test.go | 7 +- .../servicecatalog/provisioning_artifact.go | 7 +- .../servicecatalog/provisioning_parameters.go | 7 +- .../services/servicediscovery/helpers.go | 6 +- .../services/servicediscovery/instances.go | 7 +- .../services/servicediscovery/namespaces.go | 10 +- .../servicediscovery/namespaces_test.go | 7 +- .../services/servicediscovery/services.go | 10 +- .../servicediscovery/services_test.go | 7 +- .../configuration_set_event_destinations.go | 7 +- resources/services/ses/configuration_sets.go | 9 +- .../ses/configuration_sets_mock_test.go | 7 +- .../custom_verification_email_templates.go | 7 +- ..._verification_email_templates_mock_test.go | 7 +- resources/services/ses/identities.go | 11 +- .../services/ses/identities_mock_test.go | 7 +- resources/services/ses/templates.go | 9 +- resources/services/ses/templates_mock_test.go | 7 +- .../services/shield/protection_groups.go | 10 +- .../shield/protection_groups_mock_test.go | 7 +- resources/services/shield/protections.go | 9 +- .../services/shield/protections_mock_test.go | 7 +- resources/services/shield/subscriptions.go | 7 +- .../shield/subscriptions_mock_test.go | 7 +- resources/services/signer/profiles.go | 7 +- .../services/signer/profiles_mock_test.go | 7 +- resources/services/sns/subscriptions.go | 11 +- .../services/sns/subscriptions_mock_test.go | 8 +- resources/services/sns/topics.go | 11 +- resources/services/sns/topics_mock_test.go | 7 +- resources/services/sqs/queues.go | 11 +- resources/services/sqs/queues_mock_test.go | 5 +- resources/services/ssm/associations.go | 7 +- .../services/ssm/associations_mock_test.go | 7 +- .../services/ssm/compliance_summary_items.go | 7 +- .../ssm/compliance_summary_items_mock_test.go | 7 +- resources/services/ssm/document_versions.go | 7 +- resources/services/ssm/documents.go | 9 +- resources/services/ssm/documents_mock_test.go | 7 +- .../services/ssm/instance_compliance_items.go | 7 +- resources/services/ssm/instance_patches.go | 7 +- resources/services/ssm/instances.go | 7 +- resources/services/ssm/instances_mock_test.go | 7 +- resources/services/ssm/parameters.go | 10 +- .../services/ssm/parameters_mock_test.go | 7 +- .../services/ssoadmin/account_assignments.go | 7 +- .../ssoadmin/customer_managed_policies.go | 7 +- .../services/ssoadmin/inline_policies.go | 9 +- resources/services/ssoadmin/instances.go | 7 +- .../services/ssoadmin/instances_mock_test.go | 7 +- .../services/ssoadmin/managed_policies.go | 7 +- .../ssoadmin/permission_boundaries.go | 7 +- .../services/ssoadmin/permission_sets.go | 8 +- .../services/stepfunctions/activities.go | 7 +- .../services/stepfunctions/activities_test.go | 7 +- .../services/stepfunctions/executions.go | 7 +- .../stepfunctions/executions_map_runs.go | 7 +- .../executions_map_runs_executions.go | 7 +- .../services/stepfunctions/state_machines.go | 9 +- .../stepfunctions/state_machines_test.go | 7 +- resources/services/timestream/databases.go | 9 +- .../timestream/databases_mock_test.go | 7 +- resources/services/timestream/tables.go | 7 +- resources/services/transfer/servers.go | 10 +- resources/services/transfer/servers_test.go | 7 +- resources/services/waf/rule_groups.go | 12 +- .../services/waf/rule_groups_mock_test.go | 7 +- resources/services/waf/rules.go | 9 +- resources/services/waf/rules_mock_test.go | 7 +- resources/services/waf/web_acls.go | 10 +- resources/services/waf/web_acls_mock_test.go | 7 +- .../services/wafregional/rate_based_rules.go | 11 +- .../wafregional/rate_based_rules_mock_test.go | 7 +- resources/services/wafregional/rule_groups.go | 9 +- .../wafregional/rule_groups_mock_test.go | 7 +- resources/services/wafregional/rules.go | 9 +- .../services/wafregional/rules_mock_test.go | 7 +- resources/services/wafregional/web_acls.go | 10 +- .../wafregional/web_acls_mock_test.go | 7 +- resources/services/wafv2/ipsets.go | 9 +- resources/services/wafv2/ipsets_mock_test.go | 7 +- .../services/wafv2/regex_pattern_sets.go | 9 +- .../wafv2/regex_pattern_sets_mock_test.go | 7 +- resources/services/wafv2/rule_groups.go | 10 +- .../services/wafv2/rule_groups_mock_test.go | 7 +- resources/services/wafv2/web_acls.go | 12 +- .../services/wafv2/web_acls_mock_test.go | 7 +- .../lens_review_improvements.go | 7 +- .../lens_review_improvements_mock_test.go | 5 +- .../services/wellarchitected/lens_reviews.go | 7 +- .../wellarchitected/lens_reviews_mock_test.go | 5 +- resources/services/wellarchitected/lenses.go | 7 +- .../wellarchitected/lenses_mock_test.go | 7 +- .../wellarchitected/workload_milestones.go | 7 +- .../workload_milestones_mock_test.go | 5 +- .../wellarchitected/workload_shares.go | 7 +- .../workload_shares_mock_test.go | 5 +- .../services/wellarchitected/workloads.go | 7 +- .../wellarchitected/workloads_mock_test.go | 7 +- resources/services/workspaces/directories.go | 7 +- .../workspaces/directories_mock_test.go | 7 +- resources/services/workspaces/workspaces.go | 7 +- .../workspaces/workspaces_mock_test.go | 7 +- resources/services/xray/groups.go | 10 +- resources/services/xray/groups_mock_test.go | 7 +- resources/services/xray/sampling_rules.go | 10 +- .../services/xray/sampling_rules_mock_test.go | 7 +- 1253 files changed, 31840 insertions(+), 3354 deletions(-) create mode 100644 plugin-sdk/caser/caser.go create mode 100644 plugin-sdk/caser/caser_test.go create mode 100644 plugin-sdk/caser/initialisms.go create mode 100644 plugin-sdk/docs/generator.go create mode 100644 plugin-sdk/docs/generator_test.go create mode 100644 plugin-sdk/docs/json.go create mode 100644 plugin-sdk/docs/markdown.go create mode 100644 plugin-sdk/docs/templates/all_tables.md.go.tpl create mode 100644 plugin-sdk/docs/templates/all_tables_entry.md.go.tpl create mode 100644 plugin-sdk/docs/templates/table.md.go.tpl create mode 100644 plugin-sdk/docs/testdata/TestGeneratePluginDocs-JSON-__tables.json create mode 100644 plugin-sdk/docs/testdata/TestGeneratePluginDocs-Markdown-README.md create mode 100644 plugin-sdk/docs/testdata/TestGeneratePluginDocs-Markdown-incremental_table.md create mode 100644 plugin-sdk/docs/testdata/TestGeneratePluginDocs-Markdown-relation_relation_table_a.md create mode 100644 plugin-sdk/docs/testdata/TestGeneratePluginDocs-Markdown-relation_relation_table_b.md create mode 100644 plugin-sdk/docs/testdata/TestGeneratePluginDocs-Markdown-relation_table.md create mode 100644 plugin-sdk/docs/testdata/TestGeneratePluginDocs-Markdown-test_table.md create mode 100644 plugin-sdk/faker/faker.go create mode 100644 plugin-sdk/faker/faker_test.go create mode 100644 plugin-sdk/faker/options.go create mode 100644 plugin-sdk/glob/LICENSE create mode 100644 plugin-sdk/glob/README.md create mode 100644 plugin-sdk/glob/glob.go create mode 100644 plugin-sdk/glob/glob_test.go create mode 100644 plugin-sdk/helpers/grpczerolog/grpc.go create mode 100644 plugin-sdk/helpers/slice.go create mode 100644 plugin-sdk/helpers/slice_test.go create mode 100644 plugin-sdk/internal/batch/cap.go create mode 100644 plugin-sdk/internal/batch/slice.go create mode 100644 plugin-sdk/internal/batch/slice_test.go create mode 100644 plugin-sdk/internal/clients/state/v3/state.go create mode 100644 plugin-sdk/internal/clients/state/v3/state_test.go create mode 100644 plugin-sdk/internal/memdb/memdb.go create mode 100644 plugin-sdk/internal/memdb/memdb_test.go create mode 100644 plugin-sdk/internal/servers/destination/v0/destinations.go create mode 100644 plugin-sdk/internal/servers/destination/v0/schemav2tov3.go create mode 100644 plugin-sdk/internal/servers/destination/v1/convert.go create mode 100644 plugin-sdk/internal/servers/destination/v1/destination_test.go create mode 100644 plugin-sdk/internal/servers/destination/v1/destinations.go create mode 100644 plugin-sdk/internal/servers/discovery/v0/discovery.go create mode 100644 plugin-sdk/internal/servers/discovery/v0/discovery_test.go create mode 100644 plugin-sdk/internal/servers/discovery/v1/discovery.go create mode 100644 plugin-sdk/internal/servers/discovery/v1/discovery_test.go create mode 100644 plugin-sdk/internal/servers/plugin/v3/plugin.go create mode 100644 plugin-sdk/internal/servers/plugin/v3/plugin_test.go create mode 100644 plugin-sdk/message/sync_message.go create mode 100644 plugin-sdk/message/write_message.go create mode 100644 plugin-sdk/plugin/diff.go create mode 100644 plugin-sdk/plugin/meta.go create mode 100644 plugin-sdk/plugin/nulls.go create mode 100644 plugin-sdk/plugin/nulls_test.go create mode 100644 plugin-sdk/plugin/options.go create mode 100644 plugin-sdk/plugin/plugin.go create mode 100644 plugin-sdk/plugin/plugin_destination.go create mode 100644 plugin-sdk/plugin/plugin_package.go create mode 100644 plugin-sdk/plugin/plugin_read.go create mode 100644 plugin-sdk/plugin/plugin_source.go create mode 100644 plugin-sdk/plugin/plugin_test.go create mode 100644 plugin-sdk/plugin/plugin_testconn.go create mode 100644 plugin-sdk/plugin/sort.go create mode 100644 plugin-sdk/plugin/testing.go create mode 100644 plugin-sdk/plugin/testing_validation.go create mode 100644 plugin-sdk/plugin/testing_write.go create mode 100644 plugin-sdk/plugin/testing_write_delete.go create mode 100644 plugin-sdk/plugin/testing_write_insert.go create mode 100644 plugin-sdk/plugin/testing_write_migrate.go create mode 100644 plugin-sdk/plugin/testing_write_upsert.go create mode 100644 plugin-sdk/plugin/validate.go create mode 100644 plugin-sdk/premium/monitor.go create mode 100644 plugin-sdk/premium/monitor_test.go create mode 100644 plugin-sdk/premium/offline.go create mode 100644 plugin-sdk/premium/offline.key create mode 100644 plugin-sdk/premium/offline_test.go create mode 100644 plugin-sdk/premium/tables.go create mode 100644 plugin-sdk/premium/tables_test.go create mode 100644 plugin-sdk/premium/usage.go create mode 100644 plugin-sdk/premium/usage_test.go create mode 100644 plugin-sdk/scalar/LICENSE create mode 100644 plugin-sdk/scalar/README.md create mode 100644 plugin-sdk/scalar/binary.go create mode 100644 plugin-sdk/scalar/binary_test.go create mode 100644 plugin-sdk/scalar/bool.go create mode 100644 plugin-sdk/scalar/bool_test.go create mode 100644 plugin-sdk/scalar/convert.go create mode 100644 plugin-sdk/scalar/date32.go create mode 100644 plugin-sdk/scalar/date32_test.go create mode 100644 plugin-sdk/scalar/date64.go create mode 100644 plugin-sdk/scalar/date64_test.go create mode 100644 plugin-sdk/scalar/decimal.go create mode 100644 plugin-sdk/scalar/decimal_test.go create mode 100644 plugin-sdk/scalar/duration.go create mode 100644 plugin-sdk/scalar/duration_test.go create mode 100644 plugin-sdk/scalar/errors.go create mode 100644 plugin-sdk/scalar/float.go create mode 100644 plugin-sdk/scalar/float_test.go create mode 100644 plugin-sdk/scalar/inet.go create mode 100644 plugin-sdk/scalar/inet_test.go create mode 100644 plugin-sdk/scalar/int.go create mode 100644 plugin-sdk/scalar/int_test.go create mode 100644 plugin-sdk/scalar/interval.go create mode 100644 plugin-sdk/scalar/json.go create mode 100644 plugin-sdk/scalar/json_test.go create mode 100644 plugin-sdk/scalar/list.go create mode 100644 plugin-sdk/scalar/list_test.go create mode 100644 plugin-sdk/scalar/mac.go create mode 100644 plugin-sdk/scalar/mac_test.go create mode 100644 plugin-sdk/scalar/scalar.go create mode 100644 plugin-sdk/scalar/scalar_test.go create mode 100644 plugin-sdk/scalar/string.go create mode 100644 plugin-sdk/scalar/string_test.go create mode 100644 plugin-sdk/scalar/struct.go create mode 100644 plugin-sdk/scalar/struct_test.go create mode 100644 plugin-sdk/scalar/time.go create mode 100644 plugin-sdk/scalar/timestamp.go create mode 100644 plugin-sdk/scalar/timestamp_test.go create mode 100644 plugin-sdk/scalar/type_test.go create mode 100644 plugin-sdk/scalar/uint.go create mode 100644 plugin-sdk/scalar/uint_test.go create mode 100644 plugin-sdk/scalar/uuid.go create mode 100644 plugin-sdk/scalar/uuid_test.go create mode 100644 plugin-sdk/scheduler/batch.go create mode 100644 plugin-sdk/scheduler/benchmark_test.go create mode 100644 plugin-sdk/scheduler/benchmark_test.go.backup create mode 100644 plugin-sdk/scheduler/metrics.go create mode 100644 plugin-sdk/scheduler/metrics_test.go create mode 100644 plugin-sdk/scheduler/scheduler.go create mode 100644 plugin-sdk/scheduler/scheduler_dfs.go create mode 100644 plugin-sdk/scheduler/scheduler_round_robin.go create mode 100644 plugin-sdk/scheduler/scheduler_round_robin_test.go create mode 100644 plugin-sdk/scheduler/scheduler_shuffle.go create mode 100644 plugin-sdk/scheduler/scheduler_test.go create mode 100644 plugin-sdk/scheduler/strategy.go create mode 100644 plugin-sdk/scheduler/strategy_test.go create mode 100644 plugin-sdk/schema/arrow.go create mode 100644 plugin-sdk/schema/arrow_test.go create mode 100644 plugin-sdk/schema/column.go create mode 100644 plugin-sdk/schema/convert.go create mode 100644 plugin-sdk/schema/errors.go create mode 100644 plugin-sdk/schema/meta.go create mode 100644 plugin-sdk/schema/resolvers.go create mode 100644 plugin-sdk/schema/resource.go create mode 100644 plugin-sdk/schema/table.go create mode 100644 plugin-sdk/schema/table_test.go create mode 100644 plugin-sdk/schema/testdata.go create mode 100644 plugin-sdk/schema/testdata_test.go create mode 100644 plugin-sdk/schema/validators.go create mode 100644 plugin-sdk/schema/validators_test.go create mode 100644 plugin-sdk/serve/constants.go create mode 100644 plugin-sdk/serve/destination_v1_test.go create mode 100644 plugin-sdk/serve/docs.go create mode 100644 plugin-sdk/serve/docs_test.go create mode 100644 plugin-sdk/serve/enum.go create mode 100644 plugin-sdk/serve/env.go create mode 100644 plugin-sdk/serve/info.go create mode 100644 plugin-sdk/serve/opentelemetry.go create mode 100644 plugin-sdk/serve/opentelemetry_test.go create mode 100644 plugin-sdk/serve/package.go create mode 100644 plugin-sdk/serve/package_test.go create mode 100644 plugin-sdk/serve/plugin.go create mode 100644 plugin-sdk/serve/plugin_test.go create mode 100644 plugin-sdk/serve/state_test.go create mode 100644 plugin-sdk/serve/testdata/destination_spec_schema.json create mode 100644 plugin-sdk/serve/testdata/memdbtables.json create mode 100644 plugin-sdk/serve/testdata/message.txt create mode 100644 plugin-sdk/serve/testdata/source_spec_schema.json create mode 100644 plugin-sdk/state/client.go create mode 100644 plugin-sdk/transformers/ignore_in_tests.go create mode 100644 plugin-sdk/transformers/name.go create mode 100644 plugin-sdk/transformers/options.go create mode 100644 plugin-sdk/transformers/resolver.go create mode 100644 plugin-sdk/transformers/struct.go create mode 100644 plugin-sdk/transformers/struct_test.go create mode 100644 plugin-sdk/transformers/tables.go create mode 100644 plugin-sdk/transformers/type.go create mode 100644 plugin-sdk/types/extensions.go create mode 100644 plugin-sdk/types/extensions_test.go create mode 100644 plugin-sdk/types/inet.go create mode 100644 plugin-sdk/types/inet_test.go create mode 100644 plugin-sdk/types/json.go create mode 100644 plugin-sdk/types/json_test.go create mode 100644 plugin-sdk/types/mac.go create mode 100644 plugin-sdk/types/mac_test.go create mode 100644 plugin-sdk/types/register.go create mode 100644 plugin-sdk/types/uuid.go create mode 100644 plugin-sdk/types/uuid_test.go create mode 100644 plugin-sdk/writers/batchwriter/batchwriter.go create mode 100644 plugin-sdk/writers/batchwriter/batchwriter_test.go create mode 100644 plugin-sdk/writers/batchwriter/unimplemented.go create mode 100644 plugin-sdk/writers/batchwriter/unimplemented_test.go create mode 100644 plugin-sdk/writers/mixedbatchwriter/mixedbatchwriter.go create mode 100644 plugin-sdk/writers/mixedbatchwriter/mixedbatchwriter_test.go create mode 100644 plugin-sdk/writers/mixedbatchwriter/unimplemented.go create mode 100644 plugin-sdk/writers/mixedbatchwriter/unimplemented_test.go create mode 100644 plugin-sdk/writers/msgtype.go create mode 100644 plugin-sdk/writers/streamingbatchwriter/mocktimer_test.go create mode 100644 plugin-sdk/writers/streamingbatchwriter/streamingbatchwriter.go create mode 100644 plugin-sdk/writers/streamingbatchwriter/streamingbatchwriter_test.go create mode 100644 plugin-sdk/writers/streamingbatchwriter/unimplemented.go create mode 100644 plugin-sdk/writers/streamingbatchwriter/unimplemented_test.go create mode 100644 plugin-sdk/writers/ticker.go create mode 100644 plugin-sdk/writers/writers.go create mode 100644 plugin-sdk/writers/writers_test.go diff --git a/.golangci.yml b/.golangci.yml index a91cf118..8185afc2 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -15,6 +15,8 @@ linters-settings: rewrite-rules: - pattern: 'interface{}' replacement: 'any' + goimports: + local-prefixes: github.com/OpsHelmInc errcheck: check-blank: false ignore: fmt:.*,[rR]ead|[wW]rite|[cC]lose,io:Copy @@ -81,6 +83,7 @@ linters: - errcheck - gocritic - gofmt + - goimports - gosimple - govet - ineffassign diff --git a/client/account.go b/client/account.go index eb01e026..b7e9167a 100644 --- a/client/account.go +++ b/client/account.go @@ -5,8 +5,6 @@ import ( "errors" "strings" - "github.com/OpsHelmInc/cloudquery/v2/client/services" - "github.com/OpsHelmInc/cloudquery/v2/client/spec" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ec2" @@ -15,6 +13,9 @@ import ( "github.com/aws/smithy-go" "github.com/rs/zerolog" "github.com/thoas/go-funk" + + "github.com/OpsHelmInc/cloudquery/v2/client/services" + "github.com/OpsHelmInc/cloudquery/v2/client/spec" ) type svcsDetail struct { @@ -198,6 +199,7 @@ func verifyRegions(regions []string) error { } return nil } + func isAllRegions(regions []string) bool { // if regions array is not valid return false err := verifyRegions(regions) diff --git a/client/aws_sdk.go b/client/aws_sdk.go index 378f1ae1..c277b357 100644 --- a/client/aws_sdk.go +++ b/client/aws_sdk.go @@ -4,13 +4,14 @@ import ( "context" "time" - "github.com/OpsHelmInc/cloudquery/v2/client/spec" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/retry" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials/stscreds" "github.com/aws/aws-sdk-go-v2/service/sts" "github.com/rs/zerolog" + + "github.com/OpsHelmInc/cloudquery/v2/client/spec" ) func ConfigureAwsSDK(ctx context.Context, logger zerolog.Logger, awsPluginSpec *spec.Spec, account spec.Account, stsClient AssumeRoleAPIClient) (aws.Config, error) { diff --git a/client/client.go b/client/client.go index 23dfb6c2..59754595 100644 --- a/client/client.go +++ b/client/client.go @@ -7,15 +7,16 @@ import ( "strings" "sync" - "github.com/OpsHelmInc/cloudquery/v2/client/spec" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/sts" wafv2types "github.com/aws/aws-sdk-go-v2/service/wafv2/types" "github.com/aws/smithy-go/logging" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/state" "github.com/rs/zerolog" "golang.org/x/sync/errgroup" + + "github.com/OpsHelmInc/cloudquery/v2/client/spec" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/state" ) type Client struct { @@ -51,10 +52,13 @@ const ( awsCnCloudfrontScopeRegion = "cn-north-1" ) -var errInvalidRegion = errors.New("region wildcard \"*\" is only supported as first argument") -var errUnknownRegion = func(region string) error { - return fmt.Errorf("unknown region: %q", region) -} +var ( + errInvalidRegion = errors.New("region wildcard \"*\" is only supported as first argument") + errUnknownRegion = func(region string) error { + return fmt.Errorf("unknown region: %q", region) + } +) + var errRetrievingCredentials = errors.New("error retrieving AWS credentials (see logs for details). Please verify your credentials and try again") var ErrPaidAPIsNotEnabled = errors.New("not fetching resource because `use_paid_apis` is set to false") @@ -102,6 +106,7 @@ func (c *Client) updateService(service AWSServiceName) { c.ServicesManager.ServicesByPartitionAccount(c.Partition, c.AccountID).InitService(service) } } + func (c *Client) Services(service_names ...AWSServiceName) *Services { for _, service := range service_names { c.updateService(service) diff --git a/client/client_test.go b/client/client_test.go index c0eaa5ff..3d5a3b66 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -6,7 +6,6 @@ import ( "os" "testing" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" @@ -14,6 +13,8 @@ import ( "github.com/golang/mock/gomock" "github.com/google/go-cmp/cmp" "github.com/rs/zerolog" + + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" ) func Test_findEnabledRegions(t *testing.T) { diff --git a/client/columns.go b/client/columns.go index 57570c62..c50c0feb 100644 --- a/client/columns.go +++ b/client/columns.go @@ -2,7 +2,8 @@ package client import ( "github.com/apache/arrow/go/v16/arrow" - "github.com/cloudquery/plugin-sdk/v4/schema" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" ) func DefaultAccountIDColumn(pk bool) schema.Column { diff --git a/client/helpers.go b/client/helpers.go index e267a4c4..d93b8995 100644 --- a/client/helpers.go +++ b/client/helpers.go @@ -13,8 +13,9 @@ import ( "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/smithy-go" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) type AWSService string diff --git a/client/helpers_test.go b/client/helpers_test.go index 69bd3396..de1ac927 100644 --- a/client/helpers_test.go +++ b/client/helpers_test.go @@ -9,10 +9,11 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" ttypes "github.com/aws/aws-sdk-go-v2/service/acm/types" "github.com/aws/aws-sdk-go-v2/service/apigateway/types" - "github.com/cloudquery/plugin-sdk/v4/scalar" - "github.com/cloudquery/plugin-sdk/v4/schema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/scalar" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" ) func TestResolveARN(t *testing.T) { diff --git a/client/mocks/codepipeline.go b/client/mocks/codepipeline.go index 6361d2b2..0514f53f 100644 --- a/client/mocks/codepipeline.go +++ b/client/mocks/codepipeline.go @@ -335,6 +335,66 @@ func (mr *MockCodepipelineClientMockRecorder) ListPipelines(arg0, arg1 interface return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListPipelines", reflect.TypeOf((*MockCodepipelineClient)(nil).ListPipelines), varargs...) } +// ListRuleExecutions mocks base method. +func (m *MockCodepipelineClient) ListRuleExecutions(arg0 context.Context, arg1 *codepipeline.ListRuleExecutionsInput, arg2 ...func(*codepipeline.Options)) (*codepipeline.ListRuleExecutionsOutput, error) { + + // Assertion inserted by client/mockgen/main.go + o := &codepipeline.Options{} + for _, f := range arg2 { + f(o) + } + if o.Region == "" { + m.ctrl.T.Errorf("Region not set in call to ListRuleExecutions") + } + + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListRuleExecutions", varargs...) + ret0, _ := ret[0].(*codepipeline.ListRuleExecutionsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListRuleExecutions indicates an expected call of ListRuleExecutions. +func (mr *MockCodepipelineClientMockRecorder) ListRuleExecutions(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRuleExecutions", reflect.TypeOf((*MockCodepipelineClient)(nil).ListRuleExecutions), varargs...) +} + +// ListRuleTypes mocks base method. +func (m *MockCodepipelineClient) ListRuleTypes(arg0 context.Context, arg1 *codepipeline.ListRuleTypesInput, arg2 ...func(*codepipeline.Options)) (*codepipeline.ListRuleTypesOutput, error) { + + // Assertion inserted by client/mockgen/main.go + o := &codepipeline.Options{} + for _, f := range arg2 { + f(o) + } + if o.Region == "" { + m.ctrl.T.Errorf("Region not set in call to ListRuleTypes") + } + + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListRuleTypes", varargs...) + ret0, _ := ret[0].(*codepipeline.ListRuleTypesOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListRuleTypes indicates an expected call of ListRuleTypes. +func (mr *MockCodepipelineClientMockRecorder) ListRuleTypes(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRuleTypes", reflect.TypeOf((*MockCodepipelineClient)(nil).ListRuleTypes), varargs...) +} + // ListTagsForResource mocks base method. func (m *MockCodepipelineClient) ListTagsForResource(arg0 context.Context, arg1 *codepipeline.ListTagsForResourceInput, arg2 ...func(*codepipeline.Options)) (*codepipeline.ListTagsForResourceOutput, error) { diff --git a/client/mocks/ec2.go b/client/mocks/ec2.go index b36f984f..7c49fedb 100644 --- a/client/mocks/ec2.go +++ b/client/mocks/ec2.go @@ -1745,6 +1745,36 @@ func (mr *MockEc2ClientMockRecorder) DescribeIpamByoasn(arg0, arg1 interface{}, return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeIpamByoasn", reflect.TypeOf((*MockEc2Client)(nil).DescribeIpamByoasn), varargs...) } +// DescribeIpamExternalResourceVerificationTokens mocks base method. +func (m *MockEc2Client) DescribeIpamExternalResourceVerificationTokens(arg0 context.Context, arg1 *ec2.DescribeIpamExternalResourceVerificationTokensInput, arg2 ...func(*ec2.Options)) (*ec2.DescribeIpamExternalResourceVerificationTokensOutput, error) { + + // Assertion inserted by client/mockgen/main.go + o := &ec2.Options{} + for _, f := range arg2 { + f(o) + } + if o.Region == "" { + m.ctrl.T.Errorf("Region not set in call to DescribeIpamExternalResourceVerificationTokens") + } + + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DescribeIpamExternalResourceVerificationTokens", varargs...) + ret0, _ := ret[0].(*ec2.DescribeIpamExternalResourceVerificationTokensOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DescribeIpamExternalResourceVerificationTokens indicates an expected call of DescribeIpamExternalResourceVerificationTokens. +func (mr *MockEc2ClientMockRecorder) DescribeIpamExternalResourceVerificationTokens(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeIpamExternalResourceVerificationTokens", reflect.TypeOf((*MockEc2Client)(nil).DescribeIpamExternalResourceVerificationTokens), varargs...) +} + // DescribeIpamPools mocks base method. func (m *MockEc2Client) DescribeIpamPools(arg0 context.Context, arg1 *ec2.DescribeIpamPoolsInput, arg2 ...func(*ec2.Options)) (*ec2.DescribeIpamPoolsOutput, error) { diff --git a/client/mocks/ecr.go b/client/mocks/ecr.go index fd9ca264..6c6e35b0 100644 --- a/client/mocks/ecr.go +++ b/client/mocks/ecr.go @@ -275,6 +275,66 @@ func (mr *MockEcrClientMockRecorder) DescribeRepositories(arg0, arg1 interface{} return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeRepositories", reflect.TypeOf((*MockEcrClient)(nil).DescribeRepositories), varargs...) } +// DescribeRepositoryCreationTemplates mocks base method. +func (m *MockEcrClient) DescribeRepositoryCreationTemplates(arg0 context.Context, arg1 *ecr.DescribeRepositoryCreationTemplatesInput, arg2 ...func(*ecr.Options)) (*ecr.DescribeRepositoryCreationTemplatesOutput, error) { + + // Assertion inserted by client/mockgen/main.go + o := &ecr.Options{} + for _, f := range arg2 { + f(o) + } + if o.Region == "" { + m.ctrl.T.Errorf("Region not set in call to DescribeRepositoryCreationTemplates") + } + + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DescribeRepositoryCreationTemplates", varargs...) + ret0, _ := ret[0].(*ecr.DescribeRepositoryCreationTemplatesOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DescribeRepositoryCreationTemplates indicates an expected call of DescribeRepositoryCreationTemplates. +func (mr *MockEcrClientMockRecorder) DescribeRepositoryCreationTemplates(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeRepositoryCreationTemplates", reflect.TypeOf((*MockEcrClient)(nil).DescribeRepositoryCreationTemplates), varargs...) +} + +// GetAccountSetting mocks base method. +func (m *MockEcrClient) GetAccountSetting(arg0 context.Context, arg1 *ecr.GetAccountSettingInput, arg2 ...func(*ecr.Options)) (*ecr.GetAccountSettingOutput, error) { + + // Assertion inserted by client/mockgen/main.go + o := &ecr.Options{} + for _, f := range arg2 { + f(o) + } + if o.Region == "" { + m.ctrl.T.Errorf("Region not set in call to GetAccountSetting") + } + + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetAccountSetting", varargs...) + ret0, _ := ret[0].(*ecr.GetAccountSettingOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAccountSetting indicates an expected call of GetAccountSetting. +func (mr *MockEcrClientMockRecorder) GetAccountSetting(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAccountSetting", reflect.TypeOf((*MockEcrClient)(nil).GetAccountSetting), varargs...) +} + // GetAuthorizationToken mocks base method. func (m *MockEcrClient) GetAuthorizationToken(arg0 context.Context, arg1 *ecr.GetAuthorizationTokenInput, arg2 ...func(*ecr.Options)) (*ecr.GetAuthorizationTokenOutput, error) { diff --git a/client/mocks/elasticloadbalancingv2.go b/client/mocks/elasticloadbalancingv2.go index 8f1f9385..d44a7e9f 100644 --- a/client/mocks/elasticloadbalancingv2.go +++ b/client/mocks/elasticloadbalancingv2.go @@ -65,6 +65,36 @@ func (mr *MockElasticloadbalancingv2ClientMockRecorder) DescribeAccountLimits(ar return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeAccountLimits", reflect.TypeOf((*MockElasticloadbalancingv2Client)(nil).DescribeAccountLimits), varargs...) } +// DescribeListenerAttributes mocks base method. +func (m *MockElasticloadbalancingv2Client) DescribeListenerAttributes(arg0 context.Context, arg1 *elasticloadbalancingv2.DescribeListenerAttributesInput, arg2 ...func(*elasticloadbalancingv2.Options)) (*elasticloadbalancingv2.DescribeListenerAttributesOutput, error) { + + // Assertion inserted by client/mockgen/main.go + o := &elasticloadbalancingv2.Options{} + for _, f := range arg2 { + f(o) + } + if o.Region == "" { + m.ctrl.T.Errorf("Region not set in call to DescribeListenerAttributes") + } + + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DescribeListenerAttributes", varargs...) + ret0, _ := ret[0].(*elasticloadbalancingv2.DescribeListenerAttributesOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DescribeListenerAttributes indicates an expected call of DescribeListenerAttributes. +func (mr *MockElasticloadbalancingv2ClientMockRecorder) DescribeListenerAttributes(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeListenerAttributes", reflect.TypeOf((*MockElasticloadbalancingv2Client)(nil).DescribeListenerAttributes), varargs...) +} + // DescribeListenerCertificates mocks base method. func (m *MockElasticloadbalancingv2Client) DescribeListenerCertificates(arg0 context.Context, arg1 *elasticloadbalancingv2.DescribeListenerCertificatesInput, arg2 ...func(*elasticloadbalancingv2.Options)) (*elasticloadbalancingv2.DescribeListenerCertificatesOutput, error) { @@ -455,6 +485,36 @@ func (mr *MockElasticloadbalancingv2ClientMockRecorder) DescribeTrustStores(arg0 return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeTrustStores", reflect.TypeOf((*MockElasticloadbalancingv2Client)(nil).DescribeTrustStores), varargs...) } +// GetResourcePolicy mocks base method. +func (m *MockElasticloadbalancingv2Client) GetResourcePolicy(arg0 context.Context, arg1 *elasticloadbalancingv2.GetResourcePolicyInput, arg2 ...func(*elasticloadbalancingv2.Options)) (*elasticloadbalancingv2.GetResourcePolicyOutput, error) { + + // Assertion inserted by client/mockgen/main.go + o := &elasticloadbalancingv2.Options{} + for _, f := range arg2 { + f(o) + } + if o.Region == "" { + m.ctrl.T.Errorf("Region not set in call to GetResourcePolicy") + } + + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetResourcePolicy", varargs...) + ret0, _ := ret[0].(*elasticloadbalancingv2.GetResourcePolicyOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetResourcePolicy indicates an expected call of GetResourcePolicy. +func (mr *MockElasticloadbalancingv2ClientMockRecorder) GetResourcePolicy(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetResourcePolicy", reflect.TypeOf((*MockElasticloadbalancingv2Client)(nil).GetResourcePolicy), varargs...) +} + // GetTrustStoreCaCertificatesBundle mocks base method. func (m *MockElasticloadbalancingv2Client) GetTrustStoreCaCertificatesBundle(arg0 context.Context, arg1 *elasticloadbalancingv2.GetTrustStoreCaCertificatesBundleInput, arg2 ...func(*elasticloadbalancingv2.Options)) (*elasticloadbalancingv2.GetTrustStoreCaCertificatesBundleOutput, error) { diff --git a/client/mocks/glue.go b/client/mocks/glue.go index a5295530..f7b00ac6 100644 --- a/client/mocks/glue.go +++ b/client/mocks/glue.go @@ -845,6 +845,66 @@ func (mr *MockGlueClientMockRecorder) GetDataCatalogEncryptionSettings(arg0, arg return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDataCatalogEncryptionSettings", reflect.TypeOf((*MockGlueClient)(nil).GetDataCatalogEncryptionSettings), varargs...) } +// GetDataQualityModel mocks base method. +func (m *MockGlueClient) GetDataQualityModel(arg0 context.Context, arg1 *glue.GetDataQualityModelInput, arg2 ...func(*glue.Options)) (*glue.GetDataQualityModelOutput, error) { + + // Assertion inserted by client/mockgen/main.go + o := &glue.Options{} + for _, f := range arg2 { + f(o) + } + if o.Region == "" { + m.ctrl.T.Errorf("Region not set in call to GetDataQualityModel") + } + + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetDataQualityModel", varargs...) + ret0, _ := ret[0].(*glue.GetDataQualityModelOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetDataQualityModel indicates an expected call of GetDataQualityModel. +func (mr *MockGlueClientMockRecorder) GetDataQualityModel(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDataQualityModel", reflect.TypeOf((*MockGlueClient)(nil).GetDataQualityModel), varargs...) +} + +// GetDataQualityModelResult mocks base method. +func (m *MockGlueClient) GetDataQualityModelResult(arg0 context.Context, arg1 *glue.GetDataQualityModelResultInput, arg2 ...func(*glue.Options)) (*glue.GetDataQualityModelResultOutput, error) { + + // Assertion inserted by client/mockgen/main.go + o := &glue.Options{} + for _, f := range arg2 { + f(o) + } + if o.Region == "" { + m.ctrl.T.Errorf("Region not set in call to GetDataQualityModelResult") + } + + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetDataQualityModelResult", varargs...) + ret0, _ := ret[0].(*glue.GetDataQualityModelResultOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetDataQualityModelResult indicates an expected call of GetDataQualityModelResult. +func (mr *MockGlueClientMockRecorder) GetDataQualityModelResult(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDataQualityModelResult", reflect.TypeOf((*MockGlueClient)(nil).GetDataQualityModelResult), varargs...) +} + // GetDataQualityResult mocks base method. func (m *MockGlueClient) GetDataQualityResult(arg0 context.Context, arg1 *glue.GetDataQualityResultInput, arg2 ...func(*glue.Options)) (*glue.GetDataQualityResultOutput, error) { @@ -2675,6 +2735,66 @@ func (mr *MockGlueClientMockRecorder) ListDataQualityRulesets(arg0, arg1 interfa return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDataQualityRulesets", reflect.TypeOf((*MockGlueClient)(nil).ListDataQualityRulesets), varargs...) } +// ListDataQualityStatisticAnnotations mocks base method. +func (m *MockGlueClient) ListDataQualityStatisticAnnotations(arg0 context.Context, arg1 *glue.ListDataQualityStatisticAnnotationsInput, arg2 ...func(*glue.Options)) (*glue.ListDataQualityStatisticAnnotationsOutput, error) { + + // Assertion inserted by client/mockgen/main.go + o := &glue.Options{} + for _, f := range arg2 { + f(o) + } + if o.Region == "" { + m.ctrl.T.Errorf("Region not set in call to ListDataQualityStatisticAnnotations") + } + + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListDataQualityStatisticAnnotations", varargs...) + ret0, _ := ret[0].(*glue.ListDataQualityStatisticAnnotationsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListDataQualityStatisticAnnotations indicates an expected call of ListDataQualityStatisticAnnotations. +func (mr *MockGlueClientMockRecorder) ListDataQualityStatisticAnnotations(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDataQualityStatisticAnnotations", reflect.TypeOf((*MockGlueClient)(nil).ListDataQualityStatisticAnnotations), varargs...) +} + +// ListDataQualityStatistics mocks base method. +func (m *MockGlueClient) ListDataQualityStatistics(arg0 context.Context, arg1 *glue.ListDataQualityStatisticsInput, arg2 ...func(*glue.Options)) (*glue.ListDataQualityStatisticsOutput, error) { + + // Assertion inserted by client/mockgen/main.go + o := &glue.Options{} + for _, f := range arg2 { + f(o) + } + if o.Region == "" { + m.ctrl.T.Errorf("Region not set in call to ListDataQualityStatistics") + } + + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListDataQualityStatistics", varargs...) + ret0, _ := ret[0].(*glue.ListDataQualityStatisticsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListDataQualityStatistics indicates an expected call of ListDataQualityStatistics. +func (mr *MockGlueClientMockRecorder) ListDataQualityStatistics(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDataQualityStatistics", reflect.TypeOf((*MockGlueClient)(nil).ListDataQualityStatistics), varargs...) +} + // ListDevEndpoints mocks base method. func (m *MockGlueClient) ListDevEndpoints(arg0 context.Context, arg1 *glue.ListDevEndpointsInput, arg2 ...func(*glue.Options)) (*glue.ListDevEndpointsOutput, error) { diff --git a/client/mocks/iot.go b/client/mocks/iot.go index dd673dea..3b011a2f 100644 --- a/client/mocks/iot.go +++ b/client/mocks/iot.go @@ -2825,6 +2825,36 @@ func (mr *MockIotClientMockRecorder) ListRoleAliases(arg0, arg1 interface{}, arg return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRoleAliases", reflect.TypeOf((*MockIotClient)(nil).ListRoleAliases), varargs...) } +// ListSbomValidationResults mocks base method. +func (m *MockIotClient) ListSbomValidationResults(arg0 context.Context, arg1 *iot.ListSbomValidationResultsInput, arg2 ...func(*iot.Options)) (*iot.ListSbomValidationResultsOutput, error) { + + // Assertion inserted by client/mockgen/main.go + o := &iot.Options{} + for _, f := range arg2 { + f(o) + } + if o.Region == "" { + m.ctrl.T.Errorf("Region not set in call to ListSbomValidationResults") + } + + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListSbomValidationResults", varargs...) + ret0, _ := ret[0].(*iot.ListSbomValidationResultsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListSbomValidationResults indicates an expected call of ListSbomValidationResults. +func (mr *MockIotClientMockRecorder) ListSbomValidationResults(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListSbomValidationResults", reflect.TypeOf((*MockIotClient)(nil).ListSbomValidationResults), varargs...) +} + // ListScheduledAudits mocks base method. func (m *MockIotClient) ListScheduledAudits(arg0 context.Context, arg1 *iot.ListScheduledAuditsInput, arg2 ...func(*iot.Options)) (*iot.ListScheduledAuditsOutput, error) { diff --git a/client/mocks/lambda.go b/client/mocks/lambda.go index 6189c2f6..8fb0a1c4 100644 --- a/client/mocks/lambda.go +++ b/client/mocks/lambda.go @@ -305,6 +305,36 @@ func (mr *MockLambdaClientMockRecorder) GetFunctionEventInvokeConfig(arg0, arg1 return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFunctionEventInvokeConfig", reflect.TypeOf((*MockLambdaClient)(nil).GetFunctionEventInvokeConfig), varargs...) } +// GetFunctionRecursionConfig mocks base method. +func (m *MockLambdaClient) GetFunctionRecursionConfig(arg0 context.Context, arg1 *lambda.GetFunctionRecursionConfigInput, arg2 ...func(*lambda.Options)) (*lambda.GetFunctionRecursionConfigOutput, error) { + + // Assertion inserted by client/mockgen/main.go + o := &lambda.Options{} + for _, f := range arg2 { + f(o) + } + if o.Region == "" { + m.ctrl.T.Errorf("Region not set in call to GetFunctionRecursionConfig") + } + + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetFunctionRecursionConfig", varargs...) + ret0, _ := ret[0].(*lambda.GetFunctionRecursionConfigOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetFunctionRecursionConfig indicates an expected call of GetFunctionRecursionConfig. +func (mr *MockLambdaClientMockRecorder) GetFunctionRecursionConfig(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFunctionRecursionConfig", reflect.TypeOf((*MockLambdaClient)(nil).GetFunctionRecursionConfig), varargs...) +} + // GetFunctionUrlConfig mocks base method. func (m *MockLambdaClient) GetFunctionUrlConfig(arg0 context.Context, arg1 *lambda.GetFunctionUrlConfigInput, arg2 ...func(*lambda.Options)) (*lambda.GetFunctionUrlConfigOutput, error) { diff --git a/client/mocks/s3control.go b/client/mocks/s3control.go index c53f65df..a41a6dfb 100644 --- a/client/mocks/s3control.go +++ b/client/mocks/s3control.go @@ -1085,6 +1085,36 @@ func (mr *MockS3controlClientMockRecorder) ListAccessPointsForObjectLambda(arg0, return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListAccessPointsForObjectLambda", reflect.TypeOf((*MockS3controlClient)(nil).ListAccessPointsForObjectLambda), varargs...) } +// ListCallerAccessGrants mocks base method. +func (m *MockS3controlClient) ListCallerAccessGrants(arg0 context.Context, arg1 *s3control.ListCallerAccessGrantsInput, arg2 ...func(*s3control.Options)) (*s3control.ListCallerAccessGrantsOutput, error) { + + // Assertion inserted by client/mockgen/main.go + o := &s3control.Options{} + for _, f := range arg2 { + f(o) + } + if o.Region == "" { + m.ctrl.T.Errorf("Region not set in call to ListCallerAccessGrants") + } + + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListCallerAccessGrants", varargs...) + ret0, _ := ret[0].(*s3control.ListCallerAccessGrantsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListCallerAccessGrants indicates an expected call of ListCallerAccessGrants. +func (mr *MockS3controlClientMockRecorder) ListCallerAccessGrants(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListCallerAccessGrants", reflect.TypeOf((*MockS3controlClient)(nil).ListCallerAccessGrants), varargs...) +} + // ListJobs mocks base method. func (m *MockS3controlClient) ListJobs(arg0 context.Context, arg1 *s3control.ListJobsInput, arg2 ...func(*s3control.Options)) (*s3control.ListJobsOutput, error) { diff --git a/client/mocks/s3manager.go b/client/mocks/s3manager.go index ebda8337..913ffddb 100644 --- a/client/mocks/s3manager.go +++ b/client/mocks/s3manager.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/OpsHelmInc/cloudquery/client/services (interfaces: S3managerClient) +// Source: github.com/OpsHelmInc/cloudquery/v2/client/services (interfaces: S3managerClient) // Package mocks is a generated GoMock package. package mocks diff --git a/client/mocks/sagemaker.go b/client/mocks/sagemaker.go index 634ad1ba..4d2f99df 100644 --- a/client/mocks/sagemaker.go +++ b/client/mocks/sagemaker.go @@ -1445,6 +1445,36 @@ func (mr *MockSagemakerClientMockRecorder) DescribeNotebookInstanceLifecycleConf return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeNotebookInstanceLifecycleConfig", reflect.TypeOf((*MockSagemakerClient)(nil).DescribeNotebookInstanceLifecycleConfig), varargs...) } +// DescribeOptimizationJob mocks base method. +func (m *MockSagemakerClient) DescribeOptimizationJob(arg0 context.Context, arg1 *sagemaker.DescribeOptimizationJobInput, arg2 ...func(*sagemaker.Options)) (*sagemaker.DescribeOptimizationJobOutput, error) { + + // Assertion inserted by client/mockgen/main.go + o := &sagemaker.Options{} + for _, f := range arg2 { + f(o) + } + if o.Region == "" { + m.ctrl.T.Errorf("Region not set in call to DescribeOptimizationJob") + } + + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DescribeOptimizationJob", varargs...) + ret0, _ := ret[0].(*sagemaker.DescribeOptimizationJobOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DescribeOptimizationJob indicates an expected call of DescribeOptimizationJob. +func (mr *MockSagemakerClientMockRecorder) DescribeOptimizationJob(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeOptimizationJob", reflect.TypeOf((*MockSagemakerClient)(nil).DescribeOptimizationJob), varargs...) +} + // DescribePipeline mocks base method. func (m *MockSagemakerClient) DescribePipeline(arg0 context.Context, arg1 *sagemaker.DescribePipelineInput, arg2 ...func(*sagemaker.Options)) (*sagemaker.DescribePipelineOutput, error) { @@ -3755,6 +3785,36 @@ func (mr *MockSagemakerClientMockRecorder) ListNotebookInstances(arg0, arg1 inte return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListNotebookInstances", reflect.TypeOf((*MockSagemakerClient)(nil).ListNotebookInstances), varargs...) } +// ListOptimizationJobs mocks base method. +func (m *MockSagemakerClient) ListOptimizationJobs(arg0 context.Context, arg1 *sagemaker.ListOptimizationJobsInput, arg2 ...func(*sagemaker.Options)) (*sagemaker.ListOptimizationJobsOutput, error) { + + // Assertion inserted by client/mockgen/main.go + o := &sagemaker.Options{} + for _, f := range arg2 { + f(o) + } + if o.Region == "" { + m.ctrl.T.Errorf("Region not set in call to ListOptimizationJobs") + } + + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListOptimizationJobs", varargs...) + ret0, _ := ret[0].(*sagemaker.ListOptimizationJobsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListOptimizationJobs indicates an expected call of ListOptimizationJobs. +func (mr *MockSagemakerClientMockRecorder) ListOptimizationJobs(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListOptimizationJobs", reflect.TypeOf((*MockSagemakerClient)(nil).ListOptimizationJobs), varargs...) +} + // ListPipelineExecutionSteps mocks base method. func (m *MockSagemakerClient) ListPipelineExecutionSteps(arg0 context.Context, arg1 *sagemaker.ListPipelineExecutionStepsInput, arg2 ...func(*sagemaker.Options)) (*sagemaker.ListPipelineExecutionStepsOutput, error) { diff --git a/client/multiplexers.go b/client/multiplexers.go index 20a999d7..7b98e96c 100644 --- a/client/multiplexers.go +++ b/client/multiplexers.go @@ -4,7 +4,8 @@ import ( "sort" wafv2types "github.com/aws/aws-sdk-go-v2/service/wafv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" ) var AllNamespaces = []string{ // this is only used in applicationautoscaling diff --git a/client/organizations.go b/client/organizations.go index ae862719..a94504cd 100644 --- a/client/organizations.go +++ b/client/organizations.go @@ -3,16 +3,16 @@ package client import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client/services" - "github.com/OpsHelmInc/cloudquery/v2/client/spec" - "github.com/thoas/go-funk" - "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/organizations" orgTypes "github.com/aws/aws-sdk-go-v2/service/organizations/types" "github.com/aws/aws-sdk-go-v2/service/sts" "github.com/rs/zerolog" + "github.com/thoas/go-funk" + + "github.com/OpsHelmInc/cloudquery/v2/client/services" + "github.com/OpsHelmInc/cloudquery/v2/client/spec" ) // Parses org configuration and grabs the appropriate accounts diff --git a/client/organizations_test.go b/client/organizations_test.go index 33c59f0c..05341b8b 100644 --- a/client/organizations_test.go +++ b/client/organizations_test.go @@ -6,14 +6,14 @@ import ( "sort" "testing" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" - "github.com/OpsHelmInc/cloudquery/v2/client/spec" - "github.com/golang/mock/gomock" - "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/organizations" orgTypes "github.com/aws/aws-sdk-go-v2/service/organizations/types" + "github.com/golang/mock/gomock" "github.com/google/go-cmp/cmp" + + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/client/spec" ) // sets up test accounts with the following structure: diff --git a/client/resolvers.go b/client/resolvers.go index b6f13edc..14ffe257 100644 --- a/client/resolvers.go +++ b/client/resolvers.go @@ -6,12 +6,13 @@ import ( "strings" "github.com/aws/aws-sdk-go-v2/aws/arn" - "github.com/cloudquery/plugin-sdk/v4/scalar" - "github.com/cloudquery/plugin-sdk/v4/schema" "github.com/mitchellh/hashstructure/v2" "github.com/thoas/go-funk" "github.com/OpsHelmInc/ohaws" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/scalar" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" ) func ResolveAWSAccount(_ context.Context, meta schema.ClientMeta, r *schema.Resource, c schema.Column) error { diff --git a/client/resolvers_test.go b/client/resolvers_test.go index 4c019b7d..cacae3b7 100644 --- a/client/resolvers_test.go +++ b/client/resolvers_test.go @@ -7,10 +7,11 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" types1 "github.com/aws/aws-sdk-go-v2/service/codepipeline/types" types2 "github.com/aws/aws-sdk-go-v2/service/redshift/types" - "github.com/cloudquery/plugin-sdk/v4/scalar" - "github.com/cloudquery/plugin-sdk/v4/schema" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" "github.com/stretchr/testify/assert" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/scalar" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func TestResolveTags(t *testing.T) { diff --git a/client/services.go b/client/services.go index 8229661c..e724b054 100644 --- a/client/services.go +++ b/client/services.go @@ -1,7 +1,6 @@ package client import ( - "github.com/OpsHelmInc/cloudquery/v2/client/services" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/accessanalyzer" "github.com/aws/aws-sdk-go-v2/service/account" @@ -123,6 +122,8 @@ import ( "github.com/aws/aws-sdk-go-v2/service/wellarchitected" "github.com/aws/aws-sdk-go-v2/service/workspaces" "github.com/aws/aws-sdk-go-v2/service/xray" + + "github.com/OpsHelmInc/cloudquery/v2/client/services" ) func initServices(config aws.Config, regions []string) Services { diff --git a/client/services/codepipeline.go b/client/services/codepipeline.go index 24938595..0340e168 100644 --- a/client/services/codepipeline.go +++ b/client/services/codepipeline.go @@ -18,6 +18,8 @@ type CodepipelineClient interface { ListActionTypes(context.Context, *codepipeline.ListActionTypesInput, ...func(*codepipeline.Options)) (*codepipeline.ListActionTypesOutput, error) ListPipelineExecutions(context.Context, *codepipeline.ListPipelineExecutionsInput, ...func(*codepipeline.Options)) (*codepipeline.ListPipelineExecutionsOutput, error) ListPipelines(context.Context, *codepipeline.ListPipelinesInput, ...func(*codepipeline.Options)) (*codepipeline.ListPipelinesOutput, error) + ListRuleExecutions(context.Context, *codepipeline.ListRuleExecutionsInput, ...func(*codepipeline.Options)) (*codepipeline.ListRuleExecutionsOutput, error) + ListRuleTypes(context.Context, *codepipeline.ListRuleTypesInput, ...func(*codepipeline.Options)) (*codepipeline.ListRuleTypesOutput, error) ListTagsForResource(context.Context, *codepipeline.ListTagsForResourceInput, ...func(*codepipeline.Options)) (*codepipeline.ListTagsForResourceOutput, error) ListWebhooks(context.Context, *codepipeline.ListWebhooksInput, ...func(*codepipeline.Options)) (*codepipeline.ListWebhooksOutput, error) } diff --git a/client/services/ec2.go b/client/services/ec2.go index 8678e3f8..a4d7ef3c 100644 --- a/client/services/ec2.go +++ b/client/services/ec2.go @@ -65,6 +65,7 @@ type Ec2Client interface { DescribeInstances(context.Context, *ec2.DescribeInstancesInput, ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) DescribeInternetGateways(context.Context, *ec2.DescribeInternetGatewaysInput, ...func(*ec2.Options)) (*ec2.DescribeInternetGatewaysOutput, error) DescribeIpamByoasn(context.Context, *ec2.DescribeIpamByoasnInput, ...func(*ec2.Options)) (*ec2.DescribeIpamByoasnOutput, error) + DescribeIpamExternalResourceVerificationTokens(context.Context, *ec2.DescribeIpamExternalResourceVerificationTokensInput, ...func(*ec2.Options)) (*ec2.DescribeIpamExternalResourceVerificationTokensOutput, error) DescribeIpamPools(context.Context, *ec2.DescribeIpamPoolsInput, ...func(*ec2.Options)) (*ec2.DescribeIpamPoolsOutput, error) DescribeIpamResourceDiscoveries(context.Context, *ec2.DescribeIpamResourceDiscoveriesInput, ...func(*ec2.Options)) (*ec2.DescribeIpamResourceDiscoveriesOutput, error) DescribeIpamResourceDiscoveryAssociations(context.Context, *ec2.DescribeIpamResourceDiscoveryAssociationsInput, ...func(*ec2.Options)) (*ec2.DescribeIpamResourceDiscoveryAssociationsOutput, error) diff --git a/client/services/ecr.go b/client/services/ecr.go index 81ba88ee..d1f28f39 100644 --- a/client/services/ecr.go +++ b/client/services/ecr.go @@ -16,6 +16,8 @@ type EcrClient interface { DescribePullThroughCacheRules(context.Context, *ecr.DescribePullThroughCacheRulesInput, ...func(*ecr.Options)) (*ecr.DescribePullThroughCacheRulesOutput, error) DescribeRegistry(context.Context, *ecr.DescribeRegistryInput, ...func(*ecr.Options)) (*ecr.DescribeRegistryOutput, error) DescribeRepositories(context.Context, *ecr.DescribeRepositoriesInput, ...func(*ecr.Options)) (*ecr.DescribeRepositoriesOutput, error) + DescribeRepositoryCreationTemplates(context.Context, *ecr.DescribeRepositoryCreationTemplatesInput, ...func(*ecr.Options)) (*ecr.DescribeRepositoryCreationTemplatesOutput, error) + GetAccountSetting(context.Context, *ecr.GetAccountSettingInput, ...func(*ecr.Options)) (*ecr.GetAccountSettingOutput, error) GetAuthorizationToken(context.Context, *ecr.GetAuthorizationTokenInput, ...func(*ecr.Options)) (*ecr.GetAuthorizationTokenOutput, error) GetDownloadUrlForLayer(context.Context, *ecr.GetDownloadUrlForLayerInput, ...func(*ecr.Options)) (*ecr.GetDownloadUrlForLayerOutput, error) GetLifecyclePolicy(context.Context, *ecr.GetLifecyclePolicyInput, ...func(*ecr.Options)) (*ecr.GetLifecyclePolicyOutput, error) diff --git a/client/services/elasticloadbalancingv2.go b/client/services/elasticloadbalancingv2.go index 8ba7b992..c42b2f04 100644 --- a/client/services/elasticloadbalancingv2.go +++ b/client/services/elasticloadbalancingv2.go @@ -9,6 +9,7 @@ import ( //go:generate mockgen -package=mocks -destination=../mocks/elasticloadbalancingv2.go -source=elasticloadbalancingv2.go Elasticloadbalancingv2Client type Elasticloadbalancingv2Client interface { DescribeAccountLimits(context.Context, *elasticloadbalancingv2.DescribeAccountLimitsInput, ...func(*elasticloadbalancingv2.Options)) (*elasticloadbalancingv2.DescribeAccountLimitsOutput, error) + DescribeListenerAttributes(context.Context, *elasticloadbalancingv2.DescribeListenerAttributesInput, ...func(*elasticloadbalancingv2.Options)) (*elasticloadbalancingv2.DescribeListenerAttributesOutput, error) DescribeListenerCertificates(context.Context, *elasticloadbalancingv2.DescribeListenerCertificatesInput, ...func(*elasticloadbalancingv2.Options)) (*elasticloadbalancingv2.DescribeListenerCertificatesOutput, error) DescribeListeners(context.Context, *elasticloadbalancingv2.DescribeListenersInput, ...func(*elasticloadbalancingv2.Options)) (*elasticloadbalancingv2.DescribeListenersOutput, error) DescribeLoadBalancerAttributes(context.Context, *elasticloadbalancingv2.DescribeLoadBalancerAttributesInput, ...func(*elasticloadbalancingv2.Options)) (*elasticloadbalancingv2.DescribeLoadBalancerAttributesOutput, error) @@ -22,6 +23,7 @@ type Elasticloadbalancingv2Client interface { DescribeTrustStoreAssociations(context.Context, *elasticloadbalancingv2.DescribeTrustStoreAssociationsInput, ...func(*elasticloadbalancingv2.Options)) (*elasticloadbalancingv2.DescribeTrustStoreAssociationsOutput, error) DescribeTrustStoreRevocations(context.Context, *elasticloadbalancingv2.DescribeTrustStoreRevocationsInput, ...func(*elasticloadbalancingv2.Options)) (*elasticloadbalancingv2.DescribeTrustStoreRevocationsOutput, error) DescribeTrustStores(context.Context, *elasticloadbalancingv2.DescribeTrustStoresInput, ...func(*elasticloadbalancingv2.Options)) (*elasticloadbalancingv2.DescribeTrustStoresOutput, error) + GetResourcePolicy(context.Context, *elasticloadbalancingv2.GetResourcePolicyInput, ...func(*elasticloadbalancingv2.Options)) (*elasticloadbalancingv2.GetResourcePolicyOutput, error) GetTrustStoreCaCertificatesBundle(context.Context, *elasticloadbalancingv2.GetTrustStoreCaCertificatesBundleInput, ...func(*elasticloadbalancingv2.Options)) (*elasticloadbalancingv2.GetTrustStoreCaCertificatesBundleOutput, error) GetTrustStoreRevocationContent(context.Context, *elasticloadbalancingv2.GetTrustStoreRevocationContentInput, ...func(*elasticloadbalancingv2.Options)) (*elasticloadbalancingv2.GetTrustStoreRevocationContentOutput, error) } diff --git a/client/services/glue.go b/client/services/glue.go index c1b4a032..abc04182 100644 --- a/client/services/glue.go +++ b/client/services/glue.go @@ -35,6 +35,8 @@ type GlueClient interface { GetCrawlers(context.Context, *glue.GetCrawlersInput, ...func(*glue.Options)) (*glue.GetCrawlersOutput, error) GetCustomEntityType(context.Context, *glue.GetCustomEntityTypeInput, ...func(*glue.Options)) (*glue.GetCustomEntityTypeOutput, error) GetDataCatalogEncryptionSettings(context.Context, *glue.GetDataCatalogEncryptionSettingsInput, ...func(*glue.Options)) (*glue.GetDataCatalogEncryptionSettingsOutput, error) + GetDataQualityModel(context.Context, *glue.GetDataQualityModelInput, ...func(*glue.Options)) (*glue.GetDataQualityModelOutput, error) + GetDataQualityModelResult(context.Context, *glue.GetDataQualityModelResultInput, ...func(*glue.Options)) (*glue.GetDataQualityModelResultOutput, error) GetDataQualityResult(context.Context, *glue.GetDataQualityResultInput, ...func(*glue.Options)) (*glue.GetDataQualityResultOutput, error) GetDataQualityRuleRecommendationRun(context.Context, *glue.GetDataQualityRuleRecommendationRunInput, ...func(*glue.Options)) (*glue.GetDataQualityRuleRecommendationRunOutput, error) GetDataQualityRuleset(context.Context, *glue.GetDataQualityRulesetInput, ...func(*glue.Options)) (*glue.GetDataQualityRulesetOutput, error) @@ -96,6 +98,8 @@ type GlueClient interface { ListDataQualityRuleRecommendationRuns(context.Context, *glue.ListDataQualityRuleRecommendationRunsInput, ...func(*glue.Options)) (*glue.ListDataQualityRuleRecommendationRunsOutput, error) ListDataQualityRulesetEvaluationRuns(context.Context, *glue.ListDataQualityRulesetEvaluationRunsInput, ...func(*glue.Options)) (*glue.ListDataQualityRulesetEvaluationRunsOutput, error) ListDataQualityRulesets(context.Context, *glue.ListDataQualityRulesetsInput, ...func(*glue.Options)) (*glue.ListDataQualityRulesetsOutput, error) + ListDataQualityStatisticAnnotations(context.Context, *glue.ListDataQualityStatisticAnnotationsInput, ...func(*glue.Options)) (*glue.ListDataQualityStatisticAnnotationsOutput, error) + ListDataQualityStatistics(context.Context, *glue.ListDataQualityStatisticsInput, ...func(*glue.Options)) (*glue.ListDataQualityStatisticsOutput, error) ListDevEndpoints(context.Context, *glue.ListDevEndpointsInput, ...func(*glue.Options)) (*glue.ListDevEndpointsOutput, error) ListJobs(context.Context, *glue.ListJobsInput, ...func(*glue.Options)) (*glue.ListJobsOutput, error) ListMLTransforms(context.Context, *glue.ListMLTransformsInput, ...func(*glue.Options)) (*glue.ListMLTransformsOutput, error) diff --git a/client/services/iot.go b/client/services/iot.go index 3c5b187b..2c2b9284 100644 --- a/client/services/iot.go +++ b/client/services/iot.go @@ -101,6 +101,7 @@ type IotClient interface { ListProvisioningTemplates(context.Context, *iot.ListProvisioningTemplatesInput, ...func(*iot.Options)) (*iot.ListProvisioningTemplatesOutput, error) ListRelatedResourcesForAuditFinding(context.Context, *iot.ListRelatedResourcesForAuditFindingInput, ...func(*iot.Options)) (*iot.ListRelatedResourcesForAuditFindingOutput, error) ListRoleAliases(context.Context, *iot.ListRoleAliasesInput, ...func(*iot.Options)) (*iot.ListRoleAliasesOutput, error) + ListSbomValidationResults(context.Context, *iot.ListSbomValidationResultsInput, ...func(*iot.Options)) (*iot.ListSbomValidationResultsOutput, error) ListScheduledAudits(context.Context, *iot.ListScheduledAuditsInput, ...func(*iot.Options)) (*iot.ListScheduledAuditsOutput, error) ListSecurityProfiles(context.Context, *iot.ListSecurityProfilesInput, ...func(*iot.Options)) (*iot.ListSecurityProfilesOutput, error) ListSecurityProfilesForTarget(context.Context, *iot.ListSecurityProfilesForTargetInput, ...func(*iot.Options)) (*iot.ListSecurityProfilesForTargetOutput, error) diff --git a/client/services/lambda.go b/client/services/lambda.go index b5bff416..0469194a 100644 --- a/client/services/lambda.go +++ b/client/services/lambda.go @@ -17,6 +17,7 @@ type LambdaClient interface { GetFunctionConcurrency(context.Context, *lambda.GetFunctionConcurrencyInput, ...func(*lambda.Options)) (*lambda.GetFunctionConcurrencyOutput, error) GetFunctionConfiguration(context.Context, *lambda.GetFunctionConfigurationInput, ...func(*lambda.Options)) (*lambda.GetFunctionConfigurationOutput, error) GetFunctionEventInvokeConfig(context.Context, *lambda.GetFunctionEventInvokeConfigInput, ...func(*lambda.Options)) (*lambda.GetFunctionEventInvokeConfigOutput, error) + GetFunctionRecursionConfig(context.Context, *lambda.GetFunctionRecursionConfigInput, ...func(*lambda.Options)) (*lambda.GetFunctionRecursionConfigOutput, error) GetFunctionUrlConfig(context.Context, *lambda.GetFunctionUrlConfigInput, ...func(*lambda.Options)) (*lambda.GetFunctionUrlConfigOutput, error) GetLayerVersion(context.Context, *lambda.GetLayerVersionInput, ...func(*lambda.Options)) (*lambda.GetLayerVersionOutput, error) GetLayerVersionByArn(context.Context, *lambda.GetLayerVersionByArnInput, ...func(*lambda.Options)) (*lambda.GetLayerVersionByArnOutput, error) diff --git a/client/services/s3control.go b/client/services/s3control.go index 5078e3d9..bbbed273 100644 --- a/client/services/s3control.go +++ b/client/services/s3control.go @@ -43,6 +43,7 @@ type S3controlClient interface { ListAccessGrantsLocations(context.Context, *s3control.ListAccessGrantsLocationsInput, ...func(*s3control.Options)) (*s3control.ListAccessGrantsLocationsOutput, error) ListAccessPoints(context.Context, *s3control.ListAccessPointsInput, ...func(*s3control.Options)) (*s3control.ListAccessPointsOutput, error) ListAccessPointsForObjectLambda(context.Context, *s3control.ListAccessPointsForObjectLambdaInput, ...func(*s3control.Options)) (*s3control.ListAccessPointsForObjectLambdaOutput, error) + ListCallerAccessGrants(context.Context, *s3control.ListCallerAccessGrantsInput, ...func(*s3control.Options)) (*s3control.ListCallerAccessGrantsOutput, error) ListJobs(context.Context, *s3control.ListJobsInput, ...func(*s3control.Options)) (*s3control.ListJobsOutput, error) ListMultiRegionAccessPoints(context.Context, *s3control.ListMultiRegionAccessPointsInput, ...func(*s3control.Options)) (*s3control.ListMultiRegionAccessPointsOutput, error) ListRegionalBuckets(context.Context, *s3control.ListRegionalBucketsInput, ...func(*s3control.Options)) (*s3control.ListRegionalBucketsOutput, error) diff --git a/client/services/s3manager.go b/client/services/s3manager.go index 2d0bd1c7..90b3641b 100644 --- a/client/services/s3manager.go +++ b/client/services/s3manager.go @@ -2,6 +2,7 @@ package services import ( "context" + "github.com/aws/aws-sdk-go-v2/service/s3" ) diff --git a/client/services/sagemaker.go b/client/services/sagemaker.go index ede8a393..07f78d94 100644 --- a/client/services/sagemaker.go +++ b/client/services/sagemaker.go @@ -55,6 +55,7 @@ type SagemakerClient interface { DescribeMonitoringSchedule(context.Context, *sagemaker.DescribeMonitoringScheduleInput, ...func(*sagemaker.Options)) (*sagemaker.DescribeMonitoringScheduleOutput, error) DescribeNotebookInstance(context.Context, *sagemaker.DescribeNotebookInstanceInput, ...func(*sagemaker.Options)) (*sagemaker.DescribeNotebookInstanceOutput, error) DescribeNotebookInstanceLifecycleConfig(context.Context, *sagemaker.DescribeNotebookInstanceLifecycleConfigInput, ...func(*sagemaker.Options)) (*sagemaker.DescribeNotebookInstanceLifecycleConfigOutput, error) + DescribeOptimizationJob(context.Context, *sagemaker.DescribeOptimizationJobInput, ...func(*sagemaker.Options)) (*sagemaker.DescribeOptimizationJobOutput, error) DescribePipeline(context.Context, *sagemaker.DescribePipelineInput, ...func(*sagemaker.Options)) (*sagemaker.DescribePipelineOutput, error) DescribePipelineDefinitionForExecution(context.Context, *sagemaker.DescribePipelineDefinitionForExecutionInput, ...func(*sagemaker.Options)) (*sagemaker.DescribePipelineDefinitionForExecutionOutput, error) DescribePipelineExecution(context.Context, *sagemaker.DescribePipelineExecutionInput, ...func(*sagemaker.Options)) (*sagemaker.DescribePipelineExecutionOutput, error) @@ -132,6 +133,7 @@ type SagemakerClient interface { ListMonitoringSchedules(context.Context, *sagemaker.ListMonitoringSchedulesInput, ...func(*sagemaker.Options)) (*sagemaker.ListMonitoringSchedulesOutput, error) ListNotebookInstanceLifecycleConfigs(context.Context, *sagemaker.ListNotebookInstanceLifecycleConfigsInput, ...func(*sagemaker.Options)) (*sagemaker.ListNotebookInstanceLifecycleConfigsOutput, error) ListNotebookInstances(context.Context, *sagemaker.ListNotebookInstancesInput, ...func(*sagemaker.Options)) (*sagemaker.ListNotebookInstancesOutput, error) + ListOptimizationJobs(context.Context, *sagemaker.ListOptimizationJobsInput, ...func(*sagemaker.Options)) (*sagemaker.ListOptimizationJobsOutput, error) ListPipelineExecutionSteps(context.Context, *sagemaker.ListPipelineExecutionStepsInput, ...func(*sagemaker.Options)) (*sagemaker.ListPipelineExecutionStepsOutput, error) ListPipelineExecutions(context.Context, *sagemaker.ListPipelineExecutionsInput, ...func(*sagemaker.Options)) (*sagemaker.ListPipelineExecutionsOutput, error) ListPipelineParametersForExecution(context.Context, *sagemaker.ListPipelineParametersForExecutionInput, ...func(*sagemaker.Options)) (*sagemaker.ListPipelineParametersForExecutionOutput, error) diff --git a/client/spec/gen/main.go b/client/spec/gen/main.go index c31e7bae..108d9be9 100644 --- a/client/spec/gen/main.go +++ b/client/spec/gen/main.go @@ -6,8 +6,9 @@ import ( "path" "runtime" - "github.com/OpsHelmInc/cloudquery/v2/client/spec" "github.com/cloudquery/codegen/jsonschema" + + "github.com/OpsHelmInc/cloudquery/v2/client/spec" ) func main() { diff --git a/client/spec/schema.json b/client/spec/schema.json index 5d1ffd4d..87bd0915 100644 --- a/client/spec/schema.json +++ b/client/spec/schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/OpsHelmInc/cloudquery/client/spec/spec", + "$id": "https://github.com/OpsHelmInc/cloudquery/v2/client/spec/spec", "$ref": "#/$defs/Spec", "$defs": { "Account": { diff --git a/client/spec/spec.go b/client/spec/spec.go index fa1c3006..6abdcc17 100644 --- a/client/spec/spec.go +++ b/client/spec/spec.go @@ -6,9 +6,10 @@ import ( "fmt" "github.com/aws/aws-sdk-go-v2/aws" - "github.com/cloudquery/plugin-sdk/v4/scheduler" "github.com/invopop/jsonschema" orderedmap "github.com/wk8/go-ordered-map/v2" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/scheduler" ) type Spec struct { diff --git a/client/testing.go b/client/testing.go index 02b23ef2..ca1b87d7 100644 --- a/client/testing.go +++ b/client/testing.go @@ -10,14 +10,15 @@ import ( "testing" "time" - "github.com/OpsHelmInc/cloudquery/v2/client/spec" - "github.com/cloudquery/plugin-sdk/v4/message" - "github.com/cloudquery/plugin-sdk/v4/scheduler" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" "github.com/golang/mock/gomock" "github.com/rs/zerolog" + + "github.com/OpsHelmInc/cloudquery/v2/client/spec" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/scheduler" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) type TestOptions struct { diff --git a/client/transformers.go b/client/transformers.go index 8308ad66..bc5b208e 100644 --- a/client/transformers.go +++ b/client/transformers.go @@ -7,9 +7,10 @@ import ( "time" "github.com/apache/arrow/go/v16/arrow" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" "github.com/thoas/go-funk" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func TimestampTypeTransformer(field reflect.StructField) (arrow.DataType, error) { diff --git a/docs/tables/aws_athena_data_catalogs.md b/docs/tables/aws_athena_data_catalogs.md index 669d9eb2..b991ce20 100644 --- a/docs/tables/aws_athena_data_catalogs.md +++ b/docs/tables/aws_athena_data_catalogs.md @@ -24,5 +24,8 @@ The following tables depend on aws_athena_data_catalogs: |oh_resource_type|`utf8`| |name|`utf8`| |type|`utf8`| +|connection_type|`utf8`| |description|`utf8`| -|parameters|`json`| \ No newline at end of file +|error|`utf8`| +|parameters|`json`| +|status|`utf8`| \ No newline at end of file diff --git a/docs/tables/aws_backup_vaults.md b/docs/tables/aws_backup_vaults.md index 57da63a6..efba8a67 100644 --- a/docs/tables/aws_backup_vaults.md +++ b/docs/tables/aws_backup_vaults.md @@ -33,4 +33,6 @@ The following tables depend on aws_backup_vaults: |locked|`bool`| |max_retention_days|`int64`| |min_retention_days|`int64`| -|number_of_recovery_points|`int64`| \ No newline at end of file +|number_of_recovery_points|`int64`| +|vault_state|`utf8`| +|vault_type|`utf8`| \ No newline at end of file diff --git a/docs/tables/aws_batch_compute_environments.md b/docs/tables/aws_batch_compute_environments.md index 7289b0fb..07232986 100644 --- a/docs/tables/aws_batch_compute_environments.md +++ b/docs/tables/aws_batch_compute_environments.md @@ -21,6 +21,7 @@ The primary key for this table is **_cq_id**. |compute_environment_name|`utf8`| |compute_resources|`json`| |container_orchestration_type|`utf8`| +|context|`utf8`| |ecs_cluster_arn|`utf8`| |eks_configuration|`json`| |service_role|`utf8`| diff --git a/docs/tables/aws_eks_clusters.md b/docs/tables/aws_eks_clusters.md index 571226b8..55c00c92 100644 --- a/docs/tables/aws_eks_clusters.md +++ b/docs/tables/aws_eks_clusters.md @@ -43,4 +43,5 @@ The following tables depend on aws_eks_clusters: |role_arn|`utf8`| |status|`utf8`| |tags|`json`| +|upgrade_policy|`json`| |version|`utf8`| \ No newline at end of file diff --git a/docs/tables/aws_glue_connections.md b/docs/tables/aws_glue_connections.md index 2fa898a9..791bb2f0 100644 --- a/docs/tables/aws_glue_connections.md +++ b/docs/tables/aws_glue_connections.md @@ -16,6 +16,7 @@ The primary key for this table is **_cq_id**. |region|`utf8`| |arn|`utf8`| |oh_resource_type|`utf8`| +|athena_properties|`json`| |authentication_configuration|`json`| |connection_properties|`json`| |connection_type|`utf8`| diff --git a/docs/tables/aws_glue_database_tables.md b/docs/tables/aws_glue_database_tables.md index 80b95055..61443803 100644 --- a/docs/tables/aws_glue_database_tables.md +++ b/docs/tables/aws_glue_database_tables.md @@ -37,6 +37,7 @@ The following tables depend on aws_glue_database_tables: |parameters|`json`| |partition_keys|`json`| |retention|`int64`| +|status|`json`| |storage_descriptor|`json`| |table_type|`utf8`| |target_table|`json`| diff --git a/docs/tables/aws_glue_job_runs.md b/docs/tables/aws_glue_job_runs.md index fc915693..dc8e90ba 100644 --- a/docs/tables/aws_glue_job_runs.md +++ b/docs/tables/aws_glue_job_runs.md @@ -31,6 +31,7 @@ This table depends on [aws_glue_jobs](aws_glue_jobs.md). |id|`utf8`| |job_mode|`utf8`| |job_name|`utf8`| +|job_run_queuing_enabled|`bool`| |job_run_state|`utf8`| |last_modified_on|`timestamp[us, tz=UTC]`| |log_group_name|`utf8`| @@ -43,6 +44,7 @@ This table depends on [aws_glue_jobs](aws_glue_jobs.md). |profile_name|`utf8`| |security_configuration|`utf8`| |started_on|`timestamp[us, tz=UTC]`| +|state_detail|`utf8`| |timeout|`int64`| |trigger_name|`utf8`| |worker_type|`utf8`| \ No newline at end of file diff --git a/docs/tables/aws_glue_jobs.md b/docs/tables/aws_glue_jobs.md index 7d8c6e1f..feb4f646 100644 --- a/docs/tables/aws_glue_jobs.md +++ b/docs/tables/aws_glue_jobs.md @@ -33,6 +33,7 @@ The following tables depend on aws_glue_jobs: |execution_property|`json`| |glue_version|`utf8`| |job_mode|`utf8`| +|job_run_queuing_enabled|`bool`| |last_modified_on|`timestamp[us, tz=UTC]`| |log_uri|`utf8`| |maintenance_window|`utf8`| diff --git a/docs/tables/aws_lambda_function_event_source_mappings.md b/docs/tables/aws_lambda_function_event_source_mappings.md index 84db96d1..c671a484 100644 --- a/docs/tables/aws_lambda_function_event_source_mappings.md +++ b/docs/tables/aws_lambda_function_event_source_mappings.md @@ -25,9 +25,12 @@ This table depends on [aws_lambda_functions](aws_lambda_functions.md). |destination_config|`json`| |document_db_event_source_config|`json`| |event_source_arn|`utf8`| +|event_source_mapping_arn|`utf8`| |filter_criteria|`json`| +|filter_criteria_error|`json`| |function_arn|`utf8`| |function_response_types|`list`| +|kms_key_arn|`utf8`| |last_modified|`timestamp[us, tz=UTC]`| |last_processing_result|`utf8`| |maximum_batching_window_in_seconds|`int64`| diff --git a/docs/tables/aws_s3_access_grant_instances.md b/docs/tables/aws_s3_access_grant_instances.md index 95db02b7..92ba56b6 100644 --- a/docs/tables/aws_s3_access_grant_instances.md +++ b/docs/tables/aws_s3_access_grant_instances.md @@ -19,4 +19,6 @@ The primary key for this table is **_cq_id**. |access_grants_instance_arn|`utf8`| |access_grants_instance_id|`utf8`| |created_at|`timestamp[us, tz=UTC]`| -|identity_center_arn|`utf8`| \ No newline at end of file +|identity_center_application_arn|`utf8`| +|identity_center_arn|`utf8`| +|identity_center_instance_arn|`utf8`| \ No newline at end of file diff --git a/go.mod b/go.mod index 116f4e92..58dc97a5 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,6 @@ go 1.22 toolchain go1.22.1 -replace github.com/cloudquery/plugin-sdk/v4 => github.com/OpsHelmInc/cloudquery-plugin-sdk/v4 v4.59.0 - require ( github.com/OpsHelmInc/ohaws v0.5.0 github.com/apache/arrow/go/v16 v16.1.0 @@ -129,20 +127,36 @@ require ( github.com/aws/aws-sdk-go-v2/service/xray v1.26.1 github.com/aws/smithy-go v1.21.0 github.com/basgys/goxml2json v1.1.0 + github.com/bradleyjkemp/cupaloy/v2 v2.8.0 + github.com/cloudquery/cloudquery-api-go v1.11.3 github.com/cloudquery/codegen v0.3.12 - github.com/cloudquery/plugin-sdk/v4 v4.48.0 + github.com/cloudquery/plugin-pb-go v1.20.2 + github.com/cloudquery/plugin-sdk/v2 v2.7.0 github.com/gertd/go-pluralize v0.2.1 github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 + github.com/goccy/go-json v0.10.3 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.6.0 + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 github.com/invopop/jsonschema v0.12.0 github.com/mitchellh/hashstructure/v2 v2.0.2 github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.33.0 + github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 + github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.9.0 github.com/thoas/go-funk v0.9.3 github.com/wk8/go-ordered-map/v2 v2.1.8 + go.opentelemetry.io/otel v1.27.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 + go.opentelemetry.io/otel/metric v1.27.0 + go.opentelemetry.io/otel/sdk v1.27.0 + go.opentelemetry.io/otel/sdk/metric v1.27.0 + go.opentelemetry.io/otel/trace v1.27.0 + golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 golang.org/x/sync v0.8.0 ) @@ -154,7 +168,8 @@ require ( github.com/Shopify/goreferrer v0.0.0-20220729165902-8cddb4f5de06 // indirect github.com/adrg/xdg v0.4.0 // indirect github.com/andybalholm/brotli v1.1.0 // indirect - github.com/apache/arrow/go/v13 v13.0.0 // indirect + github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882 // indirect + github.com/apache/arrow/go/v14 v14.0.0-20231031200323-c49e24273160 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.18 // indirect github.com/aws/aws-sdk-go-v2/service/comprehend v1.34.3 // indirect @@ -170,9 +185,7 @@ require ( github.com/bytedance/sonic v1.11.9 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect - github.com/cloudquery/cloudquery-api-go v1.11.3 // indirect - github.com/cloudquery/plugin-pb-go v1.20.2 // indirect - github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect + github.com/cloudquery/plugin-sdk/v4 v4.17.1 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect github.com/deepmap/oapi-codegen v1.16.3 // indirect @@ -186,12 +199,10 @@ require ( github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.22.0 // indirect - github.com/goccy/go-json v0.10.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/gomarkdown/markdown v0.0.0-20240419095408-642f0ee99ae2 // indirect github.com/google/flatbuffers v24.3.25+incompatible // indirect github.com/gorilla/css v1.0.1 // indirect - github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.7 // indirect @@ -219,10 +230,8 @@ require ( github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/schollz/closestmatch v2.1.0+incompatible // indirect github.com/sirupsen/logrus v1.9.3 // indirect - github.com/spf13/cobra v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/tdewolff/minify/v2 v2.20.34 // indirect github.com/tdewolff/parse/v2 v2.7.15 // indirect @@ -238,18 +247,9 @@ require ( github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/yosssi/ace v0.0.5 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect - go.opentelemetry.io/otel v1.27.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 // indirect - go.opentelemetry.io/otel/metric v1.27.0 // indirect - go.opentelemetry.io/otel/sdk v1.27.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.27.0 // indirect - go.opentelemetry.io/otel/trace v1.27.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect golang.org/x/arch v0.8.0 // indirect golang.org/x/crypto v0.27.0 // indirect - golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 // indirect @@ -284,10 +284,10 @@ require ( golang.org/x/mod v0.18.0 // indirect golang.org/x/net v0.29.0 // indirect golang.org/x/sys v0.25.0 // indirect - golang.org/x/text v0.18.0 // indirect + golang.org/x/text v0.18.0 golang.org/x/tools v0.22.0 // indirect google.golang.org/grpc v1.67.1 - google.golang.org/protobuf v1.35.1 // indirect + google.golang.org/protobuf v1.35.1 gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 28b54058..9c5c0c8c 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,6 @@ github.com/Joker/hpp v1.0.0 h1:65+iuJYdRXv/XyN62C1uEmmOx3432rNG/rKlX6V7Kkc= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.1.3 h1:Qbeh12Vq6BxURXT1qZBRHsDxeURB8ztcL6f3EXSGeHk= github.com/Joker/jade v1.1.3/go.mod h1:T+2WLyt7VH6Lp0TRxQrUYEs64nRc83wkMQrfeIQKduM= -github.com/OpsHelmInc/cloudquery-plugin-sdk/v4 v4.59.0 h1:8y6d3gnxyFmdyVU3VkB1EfqsKVbmYcUosvVxoperSjo= -github.com/OpsHelmInc/cloudquery-plugin-sdk/v4 v4.59.0/go.mod h1:YOePv6cbAU+SIB2CpSrDmS0xWPnywbAWjL0iOML++iY= github.com/OpsHelmInc/ohaws v0.5.0 h1:4CzouJo1l8MPD+hPIFZWTvgno1M8esVnEGbqD6rkb/s= github.com/OpsHelmInc/ohaws v0.5.0/go.mod h1:u9KykoLKFkEC7N1bh+xahgIV6tybu6yNPfbcuKuL4Ow= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= @@ -31,8 +29,10 @@ github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= -github.com/apache/arrow/go/v13 v13.0.0 h1:kELrvDQuKZo8csdWYqBQfyi431x6Zs/YJTEgUuSVcWk= -github.com/apache/arrow/go/v13 v13.0.0/go.mod h1:W69eByFNO0ZR30q1/7Sr9d83zcVZmF2MiP3fFYAWJOc= +github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882 h1:mFDZW1FQk9yndPvxScp7RpcOpdSHaqcgBWO7sDlx4S8= +github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882/go.mod h1:W69eByFNO0ZR30q1/7Sr9d83zcVZmF2MiP3fFYAWJOc= +github.com/apache/arrow/go/v14 v14.0.0-20231031200323-c49e24273160 h1:SS1qi3A9YjKF2Y0AFQCNkwct4YRd0zN9ouk2x1CAyZg= +github.com/apache/arrow/go/v14 v14.0.0-20231031200323-c49e24273160/go.mod h1:TqWp9yvMb9yZSxFNiij6cmZefm+1jw3oZU0L0w9lT7E= github.com/apache/arrow/go/v16 v16.1.0 h1:dwgfOya6s03CzH9JrjCBx6bkVb4yPD4ma3haj9p7FXI= github.com/apache/arrow/go/v16 v16.1.0/go.mod h1:9wnc9mn6vEDTRIm4+27pEjQpRKuTvBaessPoEXQzxWA= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= @@ -350,6 +350,8 @@ github.com/cloudquery/plugin-pb-go v1.20.2 h1:PSNto3ohy74/CJOKqCOl+Sr+DA8D//vpax github.com/cloudquery/plugin-pb-go v1.20.2/go.mod h1:jYMI0dar0SQzK772RhJS1HvF6Q65MzsnpJ+A0N9vQEA= github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U= github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug= +github.com/cloudquery/plugin-sdk/v4 v4.17.1 h1:BQkDpWThRfqq5jKld9r7FAwfoXHV3+kMqaWTO+Wr//M= +github.com/cloudquery/plugin-sdk/v4 v4.17.1/go.mod h1:vGiAHFS2sOodpk1NC8rwuYjRp53oFuEk2mEq4MXgAJc= github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= @@ -533,12 +535,14 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= diff --git a/main.go b/main.go index 1256695e..9195f6e5 100644 --- a/main.go +++ b/main.go @@ -4,9 +4,10 @@ import ( "context" "log" - internalPlugin "github.com/OpsHelmInc/cloudquery/v2/resources/plugin" "github.com/aws/aws-sdk-go-v2/aws" - "github.com/cloudquery/plugin-sdk/v4/serve" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/serve" + internalPlugin "github.com/OpsHelmInc/cloudquery/v2/resources/plugin" ) func main() { diff --git a/plugin-sdk/caser/caser.go b/plugin-sdk/caser/caser.go new file mode 100644 index 00000000..25778e06 --- /dev/null +++ b/plugin-sdk/caser/caser.go @@ -0,0 +1,214 @@ +package caser + +import ( + "strings" + "unicode" + + "golang.org/x/text/cases" + "golang.org/x/text/language" +) + +type Caser struct { + initialisms map[string]bool + camelToSnakeExceptions map[string]string + snakeToCamelException map[string]string +} + +type Option func(*Caser) + +// WithCustomInitialisms allows specifying custom initialisms for caser. +func WithCustomInitialisms(fields map[string]bool) Option { + return func(c *Caser) { + for k, v := range fields { + c.initialisms[k] = v + } + } +} + +// WithCustomExceptions allows to specify custom exceptions for caser. +// The parameter is a map of snake:camel values like map[string]string{"oauth":"OAuth"} +func WithCustomExceptions(fields map[string]string) Option { + return func(c *Caser) { + for k, v := range fields { + c.camelToSnakeExceptions[v] = k + c.snakeToCamelException[k] = v + } + } +} + +// New creates a new instance of caser +func New(opts ...Option) *Caser { + c := &Caser{ + initialisms: make(map[string]bool), + camelToSnakeExceptions: make(map[string]string), + snakeToCamelException: make(map[string]string), + } + for k, v := range commonInitialisms { + c.initialisms[k] = v + } + for k, v := range commonExceptions { + c.snakeToCamelException[k] = v + c.camelToSnakeExceptions[v] = k + } + for _, opt := range opts { + opt(c) + } + return c +} + +// getCapWord gets the next sequence of capitalized letters as a single word. +// If there is a word after capitalized sequence it leaves one letter as beginning of the next word +func getCapWord(s string) string { + for i, r := range s { + if !unicode.IsUpper(r) { + if i == 0 { + return "" + } + return s[:i-1] + } + } + return s +} + +// ToSnake converts a given string to snake case +func (c *Caser) ToSnake(s string) string { + if s == "" { + return s + } + var result string + var words []string + var lastPos int + + s = strings.Join(strings.Fields(s), "_") + + rs := []rune(s) + + for i := 0; i < len(rs); i++ { + if i > 0 && (unicode.IsUpper(rs[i]) || rs[i] == '_') { + // check if next word is initialism + if initialism := c.startsWithInitialism(s[lastPos:]); initialism != "" { + words = append(words, initialism) + + i = lastPos + len(initialism) + lastPos = i + continue + } + + if capWord := getCapWord(s[lastPos:]); capWord != "" { + words = append(words, capWord) + + i = lastPos + len(capWord) + lastPos = i + continue + } + + words = append(words, s[lastPos:i]) + if rs[i] == '_' { + i++ + } + lastPos = i + } + } + + // append the last word + if s[lastPos:] != "" { + // handle plurals of initialisms like CDNs, ARNs, IDs + if w := s[lastPos:]; w == "s" { + words[len(words)-1] = words[len(words)-1] + w + } else { + words = append(words, s[lastPos:]) + } + } + + for k, word := range words { + if k > 0 { + result += "_" + } + + if exception, ok := c.camelToSnakeExceptions[word]; ok { + result += exception + continue + } + + result += strings.ToLower(word) + } + + return result +} + +// ToPascal returns a string converted from snake case to pascal case +func (c *Caser) ToPascal(s string) string { + if s == "" { + return s + } + result := c.ToCamel(s) + csr := cases.Title(language.Und, cases.NoLower) + return csr.String(result) +} + +// ToCamel returns a string converted from snake case to camel case +func (c *Caser) ToCamel(s string) string { + if len(s) == 0 { + return s + } + words := strings.Split(s, "_") + return strings.Join(c.capitalize(words), "") +} + +// ToTitle returns a string converted from snake case to title case. +// Title case is similar to camel case, but spaces are used in between words. +func (c *Caser) ToTitle(s string) string { + if len(s) == 0 { + return s + } + words := strings.Split(s, "_") + if _, isException := c.snakeToCamelException[strings.ToLower(words[0])]; !isException { + csr := cases.Title(language.Und, cases.NoLower) + words[0] = csr.String(words[0]) + } + return strings.Join(c.capitalize(words), " ") +} + +func (c *Caser) capitalize(words []string) []string { + n := 0 + for _, w := range words { + n += len(w) + } + var result []string + for i, word := range words { + if exception, ok := c.snakeToCamelException[word]; ok { + result = append(result, exception) + continue + } + + if i > 0 { + upper := strings.ToUpper(word) + if n > i-1 && c.initialisms[upper] { + result = append(result, upper) + continue + } + } + + if (i > 0) && len(word) > 0 { + w := []rune(word) + w[0] = unicode.ToUpper(w[0]) + result = append(result, string(w)) + } else { + result = append(result, word) + } + } + return result +} + +// startsWithInitialism returns the initialism if the given string begins with it +func (c *Caser) startsWithInitialism(s string) string { + var initialism string + // the longest initialism is 5 char, the shortest 2 + // we choose the longest match + for i := 1; i <= len(s) && i <= 5; i++ { + if len(s) > i-1 && c.initialisms[s[:i]] && len(s[:i]) > len(initialism) { + initialism = s[:i] + } + } + return initialism +} diff --git a/plugin-sdk/caser/caser_test.go b/plugin-sdk/caser/caser_test.go new file mode 100644 index 00000000..213e26a1 --- /dev/null +++ b/plugin-sdk/caser/caser_test.go @@ -0,0 +1,203 @@ +package caser + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_ToSnake(t *testing.T) { + type test struct { + Camel string + Snake string + } + + generatorTests := []test{ + {Camel: "TestCamelCase", Snake: "test_camel_case"}, + {Camel: "TestCamelCase", Snake: "test_camel_case"}, + {Camel: "AccountID", Snake: "account_id"}, + {Camel: "IDs", Snake: "ids"}, + {Camel: "PostgreSQL", Snake: "postgre_sql"}, + {Camel: "QueryStoreRetention", Snake: "query_store_retention"}, + {Camel: "TestCamelCaseLongString", Snake: "test_camel_case_long_string"}, + {Camel: "testCamelCaseLongString", Snake: "test_camel_case_long_string"}, + {Camel: "testIPv4", Snake: "test_ipv4"}, + {Camel: "CoreIPs", Snake: "core_ips"}, + {Camel: "CoreIps", Snake: "core_ips"}, + {Camel: "CoreV1", Snake: "core_v1"}, + {Camel: "APIVersion", Snake: "api_version"}, + {Camel: "TTLSecondsAfterFinished", Snake: "ttl_seconds_after_finished"}, + {Camel: "PodCIDRs", Snake: "pod_cidrs"}, + {Camel: "IAMRoles", Snake: "iam_roles"}, + {Camel: "testIAM", Snake: "test_iam"}, + {Camel: "TestAWSMode", Snake: "test_aws_mode"}, + {Camel: "Hello World", Snake: "hello_world"}, + {Camel: "Hello Wor ld", Snake: "hello_wor_ld"}, + {Camel: "Hello Wor Ld", Snake: "hello_wor_ld"}, + {Camel: "Hello wor ld", Snake: "hello_wor_ld"}, + {Camel: "Hello wOr ld", Snake: "hello_w_or_ld"}, + {Camel: "Hello wOrOr ld", Snake: "hello_w_or_or_ld"}, + {Camel: "H e l l o", Snake: "h_e_l_l_o"}, + {Camel: "H e l l o ", Snake: "h_e_l_l_o"}, + {Camel: " H e l l o", Snake: "h_e_l_l_o"}, + {Camel: " H e l l o ", Snake: "h_e_l_l_o"}, + {Camel: "Hello World", Snake: "hello_world"}, + } + t.Parallel() + c := New() + for _, tc := range generatorTests { + t.Run(tc.Camel, func(t *testing.T) { + assert.Equal(t, tc.Snake, c.ToSnake(tc.Camel)) + }) + } +} + +func Test_ToCamel(t *testing.T) { + type test struct { + Camel string + Snake string + } + + generatorTests := []test{ + {Camel: "testCamelCase", Snake: "test_camel_case"}, + {Camel: "accountID", Snake: "account_id"}, + {Camel: "arns", Snake: "arns"}, + {Camel: "postgreSQL", Snake: "postgre_sql"}, + {Camel: "queryStoreRetention", Snake: "query_store_retention"}, + {Camel: "testCamelCaseLongString", Snake: "test_camel_case_long_string"}, + {Camel: "testCamelCaseLongString", Snake: "test_camel_case_long_string"}, + {Camel: "testIPv4", Snake: "test_ipv4"}, + } + t.Parallel() + c := New() + for _, tc := range generatorTests { + t.Run(tc.Camel, func(t *testing.T) { + assert.Equal(t, tc.Camel, c.ToCamel(tc.Snake)) + }) + } +} + +func Test_ToTitle(t *testing.T) { + type test struct { + Title string + Snake string + } + + generatorTests := []test{ + {Title: "Test Camel Case", Snake: "test_camel_case"}, + {Title: "Account ID", Snake: "account_id"}, + {Title: "ARNs", Snake: "arns"}, + {Title: "Postgre SQL", Snake: "postgre_sql"}, + {Title: "Query Store Retention", Snake: "query_store_retention"}, + {Title: "Test Camel Case Long String", Snake: "test_camel_case_long_string"}, + {Title: "Test Camel Case Long String", Snake: "test_camel_case_long_string"}, + {Title: "Test IPv4", Snake: "test_ipv4"}, + {Title: "AWS Test Table", Snake: "aws_test_table"}, + {Title: "Gcp Test Table", Snake: "gcp_test_table"}, // no exception specified + } + t.Parallel() + c := New(WithCustomExceptions(map[string]string{ + "arns": "ARNs", + "aws": "AWS", + })) + for _, tc := range generatorTests { + t.Run(tc.Title, func(t *testing.T) { + assert.Equal(t, tc.Title, c.ToTitle(tc.Snake)) + }) + } +} + +func Test_ToPascal(t *testing.T) { + type test struct { + Pascal string + Snake string + } + + generatorTests := []test{ + {Pascal: "TestCamelCase", Snake: "test_camel_case"}, + {Pascal: "AccountID", Snake: "account_id"}, + {Pascal: "Arns", Snake: "arns"}, + {Pascal: "PostgreSQL", Snake: "postgre_sql"}, + {Pascal: "QueryStoreRetention", Snake: "query_store_retention"}, + {Pascal: "TestCamelCaseLongString", Snake: "test_camel_case_long_string"}, + {Pascal: "TestCamelCaseLongString", Snake: "test_camel_case_long_string"}, + {Pascal: "TestV1", Snake: "test_v1"}, + {Pascal: "TestIPv4", Snake: "test_ipv4"}, + {Pascal: "Ec2", Snake: "ec2"}, + {Pascal: "S3", Snake: "s3"}, + } + t.Parallel() + c := New() + for _, tc := range generatorTests { + t.Run(tc.Pascal, func(t *testing.T) { + assert.Equal(t, tc.Pascal, c.ToPascal(tc.Snake)) + }) + } +} + +func TestInversion(t *testing.T) { + type test struct { + Pascal string + } + + generatorTests := []test{ + {Pascal: "TestCamelCase"}, + {Pascal: "AccountID"}, + {Pascal: "Arns"}, + {Pascal: "PostgreSQL"}, + {Pascal: "QueryStoreRetention"}, + {Pascal: "TestCamelCaseLongString"}, + {Pascal: "TestCamelCaseLongString"}, + {Pascal: "TestV1"}, + {Pascal: "TestIPv4"}, + {Pascal: "TestIPv4"}, + {Pascal: "S3"}, + } + t.Parallel() + c := New() + for _, tc := range generatorTests { + t.Run(tc.Pascal, func(t *testing.T) { + assert.Equal(t, tc.Pascal, c.ToPascal(c.ToSnake(tc.Pascal))) + }) + } +} + +func Test_Configure(t *testing.T) { + type test struct { + Camel string + Snake string + } + + generatorTests := []test{ + {Camel: "CDNs", Snake: "cdns"}, + {Camel: "ARNs", Snake: "arns"}, + {Camel: "EC2", Snake: "ec2"}, + {Camel: "S3", Snake: "s3"}, + } + t.Parallel() + c := New(WithCustomInitialisms(map[string]bool{"CDN": true, "ARN": true, "EC2": true})) + for _, tc := range generatorTests { + t.Run(tc.Camel, func(t *testing.T) { + assert.Equal(t, tc.Snake, c.ToSnake(tc.Camel)) + }) + } +} + +func Test_Exceptions(t *testing.T) { + type test struct { + Camel string + Snake string + } + + generatorTests := []test{ + {Camel: "TEst", Snake: "test"}, + {Camel: "TTv2", Snake: "ttv2"}, + } + t.Parallel() + c := New(WithCustomExceptions(map[string]string{"test": "TEst", "ttv2": "TTv2"})) + for _, tc := range generatorTests { + t.Run(tc.Camel, func(t *testing.T) { + assert.Equal(t, tc.Camel, c.ToCamel(tc.Snake)) + }) + } +} diff --git a/plugin-sdk/caser/initialisms.go b/plugin-sdk/caser/initialisms.go new file mode 100644 index 00000000..eb65bfad --- /dev/null +++ b/plugin-sdk/caser/initialisms.go @@ -0,0 +1,57 @@ +package caser + +// commonInitialisms taken from https://github.com/golang/lint/blob/master/lint.go +var commonInitialisms = map[string]bool{ + "ACL": true, + "API": true, + "ASCII": true, + "CIDR": true, + "CPU": true, + "CSS": true, + "DNS": true, + "EOF": true, + "FQDN": true, + "GUID": true, + "HTML": true, + "HTTP": true, + "HTTPS": true, + "ID": true, + "IP": true, + "IPC": true, + "IPv4": true, + "IPv6": true, + "JSON": true, + "LHS": true, + "PID": true, + "QOS": true, + "QPS": true, + "RAM": true, + "RHS": true, + "RPC": true, + "SLA": true, + "SMTP": true, + "SQL": true, + "SSH": true, + "TCP": true, + "TLS": true, + "TTL": true, + "UDP": true, + "UI": true, + "UID": true, + "UUID": true, + "URI": true, + "URL": true, + "UTF8": true, + "VM": true, + "XML": true, + "XMPP": true, + "XSRF": true, + "XSS": true, +} + +// add exceptions here for things that are not automatically convertable +var commonExceptions = map[string]string{ + "oauth": "OAuth", + "ipv4": "IPv4", + "ipv6": "IPv6", +} diff --git a/plugin-sdk/docs/generator.go b/plugin-sdk/docs/generator.go new file mode 100644 index 00000000..16b0a626 --- /dev/null +++ b/plugin-sdk/docs/generator.go @@ -0,0 +1,138 @@ +package docs + +import ( + "embed" + "fmt" + "os" + "regexp" + "sort" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/caser" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +//go:embed templates/*.go.tpl +var templatesFS embed.FS + +var reMatchNewlines = regexp.MustCompile(`\n{3,}`) + +var reMatchHeaders = regexp.MustCompile(`(#{1,6}.+)\n+`) + +var DefaultTitleExceptions = map[string]string{ + // common abbreviations + "acl": "ACL", + "acls": "ACLs", + "api": "API", + "apis": "APIs", + "ca": "CA", + "cidr": "CIDR", + "cidrs": "CIDRs", + "db": "DB", + "dbs": "DBs", + "dhcp": "DHCP", + "iam": "IAM", + "iot": "IOT", + "ip": "IP", + "ips": "IPs", + "ipv4": "IPv4", + "ipv6": "IPv6", + "mfa": "MFA", + "ml": "ML", + "oauth": "OAuth", + "vpc": "VPC", + "vpcs": "VPCs", + "vpn": "VPN", + "vpns": "VPNs", + "waf": "WAF", + "wafs": "WAFs", + + // cloud providers + "aws": "AWS", + "gcp": "GCP", +} + +type Format int + +const ( + FormatMarkdown Format = iota + FormatJSON +) + +func (r Format) String() string { + return [...]string{"markdown", "json"}[r] +} + +func FormatFromString(s string) (Format, error) { + switch s { + case "markdown": + return FormatMarkdown, nil + case "json": + return FormatJSON, nil + default: + return FormatMarkdown, fmt.Errorf("unknown format %s", s) + } +} + +type Generator struct { + tables schema.Tables + titleTransformer func(*schema.Table) string + pluginName string +} + +func DefaultTitleTransformer(table *schema.Table) string { + if table.Title != "" { + return table.Title + } + csr := caser.New(caser.WithCustomExceptions(DefaultTitleExceptions)) + return csr.ToTitle(table.Name) +} + +func sortTables(tables schema.Tables) { + sort.SliceStable(tables, func(i, j int) bool { + return tables[i].Name < tables[j].Name + }) + + for _, table := range tables { + sortTables(table.Relations) + } +} + +// NewGenerator creates a new generator for the given tables. +// The tables are sorted by name. pluginName is optional and is used in markdown only +func NewGenerator(pluginName string, tables schema.Tables) *Generator { + sortedTables := make(schema.Tables, 0, len(tables)) + for _, t := range tables { + sortedTables = append(sortedTables, t.Copy(nil)) + } + sortTables(sortedTables) + + return &Generator{ + tables: sortedTables, + titleTransformer: DefaultTitleTransformer, + pluginName: pluginName, + } +} + +func (g *Generator) Generate(dir string, format Format) error { + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + return err + } + + switch format { + case FormatMarkdown: + return g.renderTablesAsMarkdown(dir) + case FormatJSON: + return g.renderTablesAsJSON(dir) + default: + return fmt.Errorf("unsupported format: %v", format) + } +} + +// setDestinationManagedCqColumns overwrites or adds the CQ columns that are managed by the destination plugins (_cq_sync_time, _cq_source_name). +// func setDestinationManagedCqColumns(tables []*schema.Table) { +// for _, table := range tables { +// table.OverwriteOrAddColumn(&schema.CqSyncTimeColumn) +// table.OverwriteOrAddColumn(&schema.CqSourceNameColumn) +// setDestinationManagedCqColumns(table.Relations) +// } +// } diff --git a/plugin-sdk/docs/generator_test.go b/plugin-sdk/docs/generator_test.go new file mode 100644 index 00000000..44e7862d --- /dev/null +++ b/plugin-sdk/docs/generator_test.go @@ -0,0 +1,164 @@ +//go:build !windows + +package docs + +import ( + "os" + "path" + "testing" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/bradleyjkemp/cupaloy/v2" + "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" +) + +var testTables = []*schema.Table{ + { + Name: "test_table", + Description: "Description for test table", + Columns: []schema.Column{ + { + Name: "int_col", + Type: arrow.PrimitiveTypes.Int64, + }, + { + Name: "id_col", + Type: arrow.PrimitiveTypes.Int64, + PrimaryKey: true, + }, + { + Name: "id_col2", + Type: arrow.PrimitiveTypes.Int64, + PrimaryKey: true, + }, + { + Name: "json_col", + Type: types.ExtensionTypes.JSON, + }, + { + Name: "list_col", + Type: arrow.ListOf(arrow.PrimitiveTypes.Int64), + }, + { + Name: "map_col", + Type: arrow.MapOf(arrow.BinaryTypes.String, arrow.PrimitiveTypes.Int64), + }, + { + Name: "struct_col", + Type: arrow.StructOf(arrow.Field{Name: "string_field", Type: arrow.BinaryTypes.String}, arrow.Field{Name: "int_field", Type: arrow.PrimitiveTypes.Int64}), + }, + }, + Relations: []*schema.Table{ + { + Name: "relation_table", + Description: "Description for relational table", + Columns: []schema.Column{ + { + Name: "string_col", + Type: arrow.BinaryTypes.String, + }, + }, + Relations: []*schema.Table{ + { + Name: "relation_relation_table_b", + Description: "Description for relational table's relation", + Columns: []schema.Column{ + { + Name: "string_col", + Type: arrow.BinaryTypes.String, + }, + }, + }, + { + Name: "relation_relation_table_a", + Description: "Description for relational table's relation", + Columns: []schema.Column{ + { + Name: "string_col", + Type: arrow.BinaryTypes.String, + }, + }, + }, + }, + }, + { + Name: "relation_table2", + Description: "Description for second relational table", + Columns: []schema.Column{ + { + Name: "string_col", + Type: arrow.BinaryTypes.String, + }, + }, + }, + }, + }, + { + Name: "incremental_table", + Description: "Description for incremental table", + IsIncremental: true, + Columns: []schema.Column{ + { + Name: "int_col", + Type: arrow.PrimitiveTypes.Int64, + }, + { + Name: "id_col", + Type: arrow.PrimitiveTypes.Int64, + PrimaryKey: true, + IncrementalKey: true, + }, + { + Name: "id_col2", + Type: arrow.PrimitiveTypes.Int64, + IncrementalKey: true, + }, + }, + }, +} + +func TestGeneratePluginDocs(t *testing.T) { + g := NewGenerator("test", testTables) + cup := cupaloy.New(cupaloy.SnapshotSubdirectory("testdata")) + + t.Run("Markdown", func(t *testing.T) { + tmpdir := t.TempDir() + + err := g.Generate(tmpdir, FormatMarkdown) + if err != nil { + t.Fatalf("unexpected error calling GeneratePluginDocs: %v", err) + } + + expectFiles := []string{"test_table.md", "relation_table.md", "relation_relation_table_a.md", "relation_relation_table_b.md", "incremental_table.md", "README.md"} + for _, exp := range expectFiles { + t.Run(exp, func(t *testing.T) { + output := path.Join(tmpdir, exp) + got, err := os.ReadFile(output) + require.NoError(t, err) + cup.SnapshotT(t, got) + }) + } + }) + + t.Run("JSON", func(t *testing.T) { + tmpdir := t.TempDir() + + err := g.Generate(tmpdir, FormatJSON) + if err != nil { + t.Fatalf("unexpected error calling GeneratePluginDocs: %v", err) + } + + expectFiles := []string{"__tables.json"} + for _, exp := range expectFiles { + t.Run(exp, func(t *testing.T) { + output := path.Join(tmpdir, exp) + got, err := os.ReadFile(output) + require.NoError(t, err) + cup.SnapshotT(t, got) + }) + } + }) +} diff --git a/plugin-sdk/docs/json.go b/plugin-sdk/docs/json.go new file mode 100644 index 00000000..0a99aa21 --- /dev/null +++ b/plugin-sdk/docs/json.go @@ -0,0 +1,64 @@ +package docs + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +type jsonTable struct { + Name string `json:"name"` + Title string `json:"title"` + Description string `json:"description"` + Columns []jsonColumn `json:"columns"` + Relations []jsonTable `json:"relations"` +} + +type jsonColumn struct { + Name string `json:"name"` + Type string `json:"type"` + IsPrimaryKey bool `json:"is_primary_key,omitempty"` + IsPrimaryKeyComponent bool `json:"is_primary_key_component,omitempty"` + IsIncrementalKey bool `json:"is_incremental_key,omitempty"` +} + +func (g *Generator) renderTablesAsJSON(dir string) error { + jsonTables := g.jsonifyTables(g.tables) + buffer := &bytes.Buffer{} + m := json.NewEncoder(buffer) + m.SetIndent("", " ") + m.SetEscapeHTML(false) + err := m.Encode(jsonTables) + if err != nil { + return err + } + outputPath := filepath.Join(dir, "__tables.json") + return os.WriteFile(outputPath, buffer.Bytes(), 0644) +} + +func (g *Generator) jsonifyTables(tables schema.Tables) []jsonTable { + jsonTables := make([]jsonTable, len(tables)) + for i, table := range tables { + jsonColumns := make([]jsonColumn, len(table.Columns)) + for c, col := range table.Columns { + jsonColumns[c] = jsonColumn{ + Name: col.Name, + Type: col.Type.String(), + IsPrimaryKey: col.PrimaryKey, + IsPrimaryKeyComponent: col.PrimaryKeyComponent, + IsIncrementalKey: col.IncrementalKey, + } + } + jsonTables[i] = jsonTable{ + Name: table.Name, + Title: g.titleTransformer(table), + Description: table.Description, + Columns: jsonColumns, + Relations: g.jsonifyTables(table.Relations), + } + } + return jsonTables +} diff --git a/plugin-sdk/docs/markdown.go b/plugin-sdk/docs/markdown.go new file mode 100644 index 00000000..f100acd2 --- /dev/null +++ b/plugin-sdk/docs/markdown.go @@ -0,0 +1,96 @@ +package docs + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "text/template" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +type templateData struct { + PluginName string + Tables schema.Tables +} + +func (g *Generator) renderTablesAsMarkdown(dir string) error { + for _, table := range g.tables { + if err := g.renderAllTables(dir, table); err != nil { + return err + } + } + t, err := template.New("all_tables.md.go.tpl").Funcs(template.FuncMap{ + "indentToDepth": indentToDepth, + }).ParseFS(templatesFS, "templates/all_tables*.md.go.tpl") + if err != nil { + return fmt.Errorf("failed to parse template for README.md: %v", err) + } + + var b bytes.Buffer + if err := t.Execute(&b, templateData{PluginName: g.pluginName, Tables: g.tables}); err != nil { + return fmt.Errorf("failed to execute template: %v", err) + } + content := formatMarkdown(b.String()) + outputPath := filepath.Join(dir, "README.md") + f, err := os.Create(outputPath) + if err != nil { + return fmt.Errorf("failed to create file %v: %v", outputPath, err) + } + defer f.Close() + f.WriteString(content) + return nil +} + +func (g *Generator) renderAllTables(dir string, t *schema.Table) error { + if err := g.renderTable(dir, t); err != nil { + return err + } + for _, r := range t.Relations { + if err := g.renderAllTables(dir, r); err != nil { + return err + } + } + return nil +} + +func (g *Generator) renderTable(dir string, table *schema.Table) error { + t := template.New("").Funcs(map[string]any{ + "title": g.titleTransformer, + }) + t, err := t.New("table.md.go.tpl").ParseFS(templatesFS, "templates/table.md.go.tpl") + if err != nil { + return fmt.Errorf("failed to parse template: %v", err) + } + + outputPath := filepath.Join(dir, fmt.Sprintf("%s.md", table.Name)) + + var b bytes.Buffer + if err := t.Execute(&b, table); err != nil { + return fmt.Errorf("failed to execute template: %v", err) + } + content := formatMarkdown(b.String()) + f, err := os.Create(outputPath) + if err != nil { + return fmt.Errorf("failed to create file %v: %v", outputPath, err) + } + defer f.Close() + f.WriteString(content) + return f.Close() +} + +func formatMarkdown(s string) string { + s = reMatchNewlines.ReplaceAllString(s, "\n\n") + return reMatchHeaders.ReplaceAllString(s, `$1`+"\n\n") +} + +func indentToDepth(table *schema.Table) string { + s := "" + t := table + for t.Parent != nil { + s += " " + t = t.Parent + } + return s +} diff --git a/plugin-sdk/docs/templates/all_tables.md.go.tpl b/plugin-sdk/docs/templates/all_tables.md.go.tpl new file mode 100644 index 00000000..008afb66 --- /dev/null +++ b/plugin-sdk/docs/templates/all_tables.md.go.tpl @@ -0,0 +1,5 @@ +# Source Plugin: {{.PluginName}} +## Tables +{{- range $table := $.Tables }} +{{- template "all_tables_entry.md.go.tpl" $table}} +{{- end }} \ No newline at end of file diff --git a/plugin-sdk/docs/templates/all_tables_entry.md.go.tpl b/plugin-sdk/docs/templates/all_tables_entry.md.go.tpl new file mode 100644 index 00000000..6166b198 --- /dev/null +++ b/plugin-sdk/docs/templates/all_tables_entry.md.go.tpl @@ -0,0 +1,5 @@ + +{{. | indentToDepth}}- [{{.Name}}]({{.Name}}.md){{ if .IsIncremental}} (Incremental){{ end }} +{{- range $index, $rel := .Relations}} +{{- template "all_tables_entry.md.go.tpl" $rel}} +{{- end}} \ No newline at end of file diff --git a/plugin-sdk/docs/templates/table.md.go.tpl b/plugin-sdk/docs/templates/table.md.go.tpl new file mode 100644 index 00000000..202d343e --- /dev/null +++ b/plugin-sdk/docs/templates/table.md.go.tpl @@ -0,0 +1,44 @@ +# Table: {{$.Name}} + +This table shows data for {{.|title}}. + +{{ $.Description }} +{{ $length := len $.PrimaryKeys -}} +{{ if eq $length 1 }} +The primary key for this table is **{{ index $.PrimaryKeys 0 }}**. +{{ else }} +The composite primary key for this table is ({{ range $index, $pk := $.PrimaryKeys -}} + {{if $index }}, {{end -}} + **{{$pk}}** + {{- end -}}). +{{ end }} +{{- if $.IsIncremental -}} +It supports incremental syncs +{{- $ikLength := len $.IncrementalKeys -}} +{{- if eq $ikLength 1 }} based on the **{{ index $.IncrementalKeys 0 }}** column +{{- else if gt $ikLength 1 }} based on the ({{ range $index, $pk := $.IncrementalKeys -}} + {{- if $index -}}, {{end -}} + **{{$pk}}** + {{- end -}}) columns +{{- end -}}. +{{- end -}} + +{{- if or ($.Relations) ($.Parent) }} +## Relations +{{- end }} +{{- if $.Parent }} +This table depends on [{{ $.Parent.Name }}]({{ $.Parent.Name }}.md). +{{- end}} +{{ if $.Relations }} +The following tables depend on {{.Name}}: +{{- range $rel := $.Relations }} + - [{{ $rel.Name }}]({{ $rel.Name }}.md) +{{- end }} +{{- end }} + +## Columns +| Name | Type | +| ------------- | ------------- | +{{- range $column := $.Columns }} +|{{$column.Name}}{{if $column.PrimaryKey}} (PK){{end}}{{if $column.IncrementalKey}} (Incremental Key){{end}}|`{{$column.Type}}`| +{{- end }} \ No newline at end of file diff --git a/plugin-sdk/docs/testdata/TestGeneratePluginDocs-JSON-__tables.json b/plugin-sdk/docs/testdata/TestGeneratePluginDocs-JSON-__tables.json new file mode 100644 index 00000000..2623746c --- /dev/null +++ b/plugin-sdk/docs/testdata/TestGeneratePluginDocs-JSON-__tables.json @@ -0,0 +1,114 @@ +[ + { + "name": "incremental_table", + "title": "Incremental Table", + "description": "Description for incremental table", + "columns": [ + { + "name": "int_col", + "type": "int64" + }, + { + "name": "id_col", + "type": "int64", + "is_primary_key": true, + "is_incremental_key": true + }, + { + "name": "id_col2", + "type": "int64", + "is_incremental_key": true + } + ], + "relations": [] + }, + { + "name": "test_table", + "title": "Test Table", + "description": "Description for test table", + "columns": [ + { + "name": "int_col", + "type": "int64" + }, + { + "name": "id_col", + "type": "int64", + "is_primary_key": true + }, + { + "name": "id_col2", + "type": "int64", + "is_primary_key": true + }, + { + "name": "json_col", + "type": "json" + }, + { + "name": "list_col", + "type": "list" + }, + { + "name": "map_col", + "type": "map" + }, + { + "name": "struct_col", + "type": "struct" + } + ], + "relations": [ + { + "name": "relation_table", + "title": "Relation Table", + "description": "Description for relational table", + "columns": [ + { + "name": "string_col", + "type": "utf8" + } + ], + "relations": [ + { + "name": "relation_relation_table_a", + "title": "Relation Relation Table A", + "description": "Description for relational table's relation", + "columns": [ + { + "name": "string_col", + "type": "utf8" + } + ], + "relations": [] + }, + { + "name": "relation_relation_table_b", + "title": "Relation Relation Table B", + "description": "Description for relational table's relation", + "columns": [ + { + "name": "string_col", + "type": "utf8" + } + ], + "relations": [] + } + ] + }, + { + "name": "relation_table2", + "title": "Relation Table2", + "description": "Description for second relational table", + "columns": [ + { + "name": "string_col", + "type": "utf8" + } + ], + "relations": [] + } + ] + } +] + diff --git a/plugin-sdk/docs/testdata/TestGeneratePluginDocs-Markdown-README.md b/plugin-sdk/docs/testdata/TestGeneratePluginDocs-Markdown-README.md new file mode 100644 index 00000000..9480a059 --- /dev/null +++ b/plugin-sdk/docs/testdata/TestGeneratePluginDocs-Markdown-README.md @@ -0,0 +1,10 @@ +# Source Plugin: test + +## Tables + +- [incremental_table](incremental_table.md) (Incremental) +- [test_table](test_table.md) + - [relation_table](relation_table.md) + - [relation_relation_table_a](relation_relation_table_a.md) + - [relation_relation_table_b](relation_relation_table_b.md) + - [relation_table2](relation_table2.md) diff --git a/plugin-sdk/docs/testdata/TestGeneratePluginDocs-Markdown-incremental_table.md b/plugin-sdk/docs/testdata/TestGeneratePluginDocs-Markdown-incremental_table.md new file mode 100644 index 00000000..b467987e --- /dev/null +++ b/plugin-sdk/docs/testdata/TestGeneratePluginDocs-Markdown-incremental_table.md @@ -0,0 +1,16 @@ +# Table: incremental_table + +This table shows data for Incremental Table. + +Description for incremental table + +The primary key for this table is **id_col**. +It supports incremental syncs based on the (**id_col**, **id_col2**) columns. + +## Columns + +| Name | Type | +| ------------- | ------------- | +|int_col|`int64`| +|id_col (PK) (Incremental Key)|`int64`| +|id_col2 (Incremental Key)|`int64`| diff --git a/plugin-sdk/docs/testdata/TestGeneratePluginDocs-Markdown-relation_relation_table_a.md b/plugin-sdk/docs/testdata/TestGeneratePluginDocs-Markdown-relation_relation_table_a.md new file mode 100644 index 00000000..80f970b1 --- /dev/null +++ b/plugin-sdk/docs/testdata/TestGeneratePluginDocs-Markdown-relation_relation_table_a.md @@ -0,0 +1,17 @@ +# Table: relation_relation_table_a + +This table shows data for Relation Relation Table A. + +Description for relational table's relation + +The composite primary key for this table is (). + +## Relations + +This table depends on [relation_table](relation_table.md). + +## Columns + +| Name | Type | +| ------------- | ------------- | +|string_col|`utf8`| diff --git a/plugin-sdk/docs/testdata/TestGeneratePluginDocs-Markdown-relation_relation_table_b.md b/plugin-sdk/docs/testdata/TestGeneratePluginDocs-Markdown-relation_relation_table_b.md new file mode 100644 index 00000000..cfc53d2c --- /dev/null +++ b/plugin-sdk/docs/testdata/TestGeneratePluginDocs-Markdown-relation_relation_table_b.md @@ -0,0 +1,17 @@ +# Table: relation_relation_table_b + +This table shows data for Relation Relation Table B. + +Description for relational table's relation + +The composite primary key for this table is (). + +## Relations + +This table depends on [relation_table](relation_table.md). + +## Columns + +| Name | Type | +| ------------- | ------------- | +|string_col|`utf8`| diff --git a/plugin-sdk/docs/testdata/TestGeneratePluginDocs-Markdown-relation_table.md b/plugin-sdk/docs/testdata/TestGeneratePluginDocs-Markdown-relation_table.md new file mode 100644 index 00000000..1cc28575 --- /dev/null +++ b/plugin-sdk/docs/testdata/TestGeneratePluginDocs-Markdown-relation_table.md @@ -0,0 +1,21 @@ +# Table: relation_table + +This table shows data for Relation Table. + +Description for relational table + +The composite primary key for this table is (). + +## Relations + +This table depends on [test_table](test_table.md). + +The following tables depend on relation_table: + - [relation_relation_table_a](relation_relation_table_a.md) + - [relation_relation_table_b](relation_relation_table_b.md) + +## Columns + +| Name | Type | +| ------------- | ------------- | +|string_col|`utf8`| diff --git a/plugin-sdk/docs/testdata/TestGeneratePluginDocs-Markdown-test_table.md b/plugin-sdk/docs/testdata/TestGeneratePluginDocs-Markdown-test_table.md new file mode 100644 index 00000000..e4a96ab3 --- /dev/null +++ b/plugin-sdk/docs/testdata/TestGeneratePluginDocs-Markdown-test_table.md @@ -0,0 +1,25 @@ +# Table: test_table + +This table shows data for Test Table. + +Description for test table + +The composite primary key for this table is (**id_col**, **id_col2**). + +## Relations + +The following tables depend on test_table: + - [relation_table](relation_table.md) + - [relation_table2](relation_table2.md) + +## Columns + +| Name | Type | +| ------------- | ------------- | +|int_col|`int64`| +|id_col (PK)|`int64`| +|id_col2 (PK)|`int64`| +|json_col|`json`| +|list_col|`list`| +|map_col|`map`| +|struct_col|`struct`| diff --git a/plugin-sdk/faker/faker.go b/plugin-sdk/faker/faker.go new file mode 100644 index 00000000..00c3dce1 --- /dev/null +++ b/plugin-sdk/faker/faker.go @@ -0,0 +1,201 @@ +package faker + +import ( + "encoding/json" + "fmt" + "math/rand" + "reflect" + "time" + + "github.com/rs/zerolog" +) + +type faker struct { + maxDepth int + logger zerolog.Logger +} + +var errEFaceNotAllowed = fmt.Errorf("any not allowed") + +func (f faker) getFakedValue(a any) (reflect.Value, error) { + t := reflect.TypeOf(a) + if t == nil { + return reflect.Value{}, errEFaceNotAllowed + } + f.maxDepth-- + if f.maxDepth < 0 { + return reflect.Value{}, fmt.Errorf("max_depth reached") + } + k := t.Kind() + switch k { + case reflect.Ptr: + v := reflect.New(t.Elem()) + var val reflect.Value + var err error + if a != reflect.Zero(reflect.TypeOf(a)).Interface() { + val, err = f.getFakedValue(reflect.ValueOf(a).Elem().Interface()) + } else { + val, err = f.getFakedValue(v.Elem().Interface()) + } + if err != nil { + return reflect.Value{}, err + } + v.Elem().Set(val.Convert(t.Elem())) + return v, nil + case reflect.Struct: + switch t.String() { + case "time.Time": + ft := time.Now().Add(time.Duration(rand.Int63())) + return reflect.ValueOf(ft), nil + default: + cur := reflect.ValueOf(a) + + v := reflect.New(t).Elem() + for i := 0; i < v.NumField(); i++ { + if !v.Field(i).CanSet() { + continue // to avoid panic to set on unexported field in struct + } + curField := cur.Field(i) + if curField.IsValid() && !curField.IsZero() { + v.Field(i).Set(curField) // preserve non-empty field values + continue + } + + val, err := f.getFakedValue(v.Field(i).Interface()) + if err != nil { + if err == errEFaceNotAllowed { // skip empty interfaces + continue + } + + f.logger.Err(err).Str("field_name", v.Type().Field(i).Name).Msg("faker: error while faking struct") + continue + } + val = val.Convert(v.Field(i).Type()) + v.Field(i).Set(val) + } + return v, nil + } + case reflect.String: + if t == reflect.TypeOf(json.Number("")) { + return reflect.ValueOf("123456789"), nil + } + return reflect.ValueOf("test string"), nil + case reflect.Slice: + switch t.String() { + case "net.IP": + sliceLen := 4 + v := reflect.MakeSlice(reflect.TypeOf([]uint8{uint8(123)}), sliceLen, sliceLen) + for i := 0; i < v.Len(); i++ { + val := reflect.ValueOf(uint8(1)) + val = val.Convert(v.Index(i).Type()) + v.Index(i).Set(val) + } + return v, nil + default: + sliceLen := 1 + v := reflect.MakeSlice(t, sliceLen, sliceLen) + for i := 0; i < v.Len(); i++ { + val, err := f.getFakedValue(v.Index(i).Interface()) + if err != nil { + return reflect.Value{}, err + } + val = val.Convert(v.Index(i).Type()) + v.Index(i).Set(val) + } + return v, nil + } + case reflect.Array: + v := reflect.New(t).Elem() + for i := 0; i < v.Len(); i++ { + val, err := f.getFakedValue(v.Index(i).Interface()) + if err != nil { + return reflect.Value{}, err + } + val = val.Convert(v.Index(i).Type()) + v.Index(i).Set(val) + } + return v, nil + case reflect.Int: + return reflect.ValueOf(int(123)), nil + case reflect.Int8: + return reflect.ValueOf(int8(123)), nil + case reflect.Int16: + return reflect.ValueOf(int16(123)), nil + case reflect.Int32: + return reflect.ValueOf(int32(123)), nil + case reflect.Int64: + return reflect.ValueOf(int64(123)), nil + case reflect.Float32: + return reflect.ValueOf(float32(123)), nil + case reflect.Float64: + return reflect.ValueOf(float64(1.123)), nil + case reflect.Bool: + return reflect.ValueOf(true), nil + + case reflect.Uint: + return reflect.ValueOf(uint(123)), nil + + case reflect.Uint8: + return reflect.ValueOf(uint8(123)), nil + + case reflect.Uint16: + return reflect.ValueOf(uint16(123)), nil + + case reflect.Uint32: + return reflect.ValueOf(uint32(123)), nil + + case reflect.Uint64: + return reflect.ValueOf(uint64(123)), nil + + case reflect.Map: + v := reflect.MakeMap(t) + for i := 0; i < 1; i++ { + keyInstance := reflect.New(t.Key()).Elem().Interface() + key, err := f.getFakedValue(keyInstance) + if err != nil { + return reflect.Value{}, err + } + key = key.Convert(t.Key()) + + valueInstance := reflect.New(t.Elem()).Elem().Interface() + val, err := f.getFakedValue(valueInstance) + if err != nil { + return reflect.Value{}, err + } + val = val.Convert(v.Type().Elem()) + v.SetMapIndex(key, val) + } + return v, nil + default: + err := fmt.Errorf("no support for kind %+v", t) + return reflect.Value{}, err + } +} + +func FakeObject(obj any, opts ...Option) error { + reflectType := reflect.TypeOf(obj) + + if reflectType.Kind() != reflect.Ptr { + return fmt.Errorf("object is not a pointer") + } + + if reflect.ValueOf(obj).IsNil() { + return fmt.Errorf("object is nil %s", reflectType.Elem().String()) + } + f := &faker{ + maxDepth: 12, + logger: zerolog.Nop(), + } + for _, o := range opts { + o(f) + } + + rval := reflect.ValueOf(obj) + finalValue, err := f.getFakedValue(obj) + if err != nil { + return err + } + + rval.Elem().Set(finalValue.Elem().Convert(reflectType.Elem())) + return nil +} diff --git a/plugin-sdk/faker/faker_test.go b/plugin-sdk/faker/faker_test.go new file mode 100644 index 00000000..f21e1640 --- /dev/null +++ b/plugin-sdk/faker/faker_test.go @@ -0,0 +1,99 @@ +package faker + +import ( + "encoding/json" + "net" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +type testFakerStruct struct { + A int + B string + C *string + D time.Time + E any + F json.Number +} + +func TestFaker(t *testing.T) { + a := testFakerStruct{} + if err := FakeObject(&a); err != nil { + t.Fatal(err) + } + assert.NotEmpty(t, a.A) + assert.NotEmpty(t, a.B) + assert.NotEmpty(t, a.C) + assert.NotEmpty(t, a.D) + assert.Empty(t, a.E) // empty interfaces are not faked + assert.Equal(t, json.Number("123456789"), a.F) // json numbers should be numbers +} + +type customType string + +type fakerStructWithCustomType struct { + A customType + B map[string]customType + C map[customType]string + D []customType +} + +func TestFakerWithCustomType(t *testing.T) { + a := fakerStructWithCustomType{} + if err := FakeObject(&a); err != nil { + t.Fatal(err) + } + assert.NotEmpty(t, a.A) + assert.NotEmpty(t, a.B) + assert.NotEmpty(t, a.C) + assert.NotEmpty(t, a.D) +} + +func TestFakerWithCustomTypePreserve(t *testing.T) { + a := fakerStructWithCustomType{ + A: "A", + B: map[string]customType{"b": "B"}, + C: map[customType]string{"c": "C"}, + D: []customType{"D", "D2"}, + } + if err := FakeObject(&a); err != nil { + t.Fatal(err) + } + assert.EqualValues(t, "A", a.A) + assert.EqualValues(t, map[string]customType{"b": "B"}, a.B) + assert.EqualValues(t, map[customType]string{"c": "C"}, a.C) + assert.EqualValues(t, []customType{"D", "D2"}, a.D) +} + +type complexType struct { + IPAddress net.IP + IPAddresses []net.IP + PtrIPAddress *net.IP + PtrIPAddresses []*net.IP + NestedComplex struct { + IPAddress net.IP + } +} + +func TestFakerCanFakeNetIP(t *testing.T) { + a := complexType{} + if err := FakeObject(&a); err != nil { + t.Fatal(err) + } + assert.NotEmpty(t, a.IPAddress) + assert.Equal(t, "1.1.1.1", a.IPAddress.String()) + + assert.Equal(t, 1, len(a.IPAddresses)) + assert.Equal(t, "1.1.1.1", a.IPAddresses[0].String()) + + assert.NotEmpty(t, a.PtrIPAddress) + assert.Equal(t, "1.1.1.1", a.PtrIPAddress.String()) + + assert.Equal(t, 1, len(a.PtrIPAddresses)) + assert.Equal(t, "1.1.1.1", a.PtrIPAddresses[0].String()) + + assert.NotEmpty(t, a.NestedComplex.IPAddress) + assert.Equal(t, "1.1.1.1", a.NestedComplex.IPAddress.String()) +} diff --git a/plugin-sdk/faker/options.go b/plugin-sdk/faker/options.go new file mode 100644 index 00000000..3a36973f --- /dev/null +++ b/plugin-sdk/faker/options.go @@ -0,0 +1,17 @@ +package faker + +import "github.com/rs/zerolog" + +type Option func(*faker) + +func WithMaxDepth(depth int) Option { + return func(f *faker) { + f.maxDepth = depth + } +} + +func WithLogger(logger zerolog.Logger) Option { + return func(f *faker) { + f.logger = logger + } +} diff --git a/plugin-sdk/glob/LICENSE b/plugin-sdk/glob/LICENSE new file mode 100644 index 00000000..a9f1724a --- /dev/null +++ b/plugin-sdk/glob/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Ryan Uber + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/plugin-sdk/glob/README.md b/plugin-sdk/glob/README.md new file mode 100644 index 00000000..380f61d5 --- /dev/null +++ b/plugin-sdk/glob/README.md @@ -0,0 +1,3 @@ +# Glob Matching Library + +This glob-matching library was copied from [ryanuber/go-glob](https://github.com/ryanuber/go-glob) and therefore falls under its [license](LICENSE). diff --git a/plugin-sdk/glob/glob.go b/plugin-sdk/glob/glob.go new file mode 100644 index 00000000..b4fd6535 --- /dev/null +++ b/plugin-sdk/glob/glob.go @@ -0,0 +1,70 @@ +package glob + +import "strings" + +// The character which is treated like a glob +const GLOB = "*" + +func IncludeTable(name string, tables []string, skipTables []string) bool { + for _, t := range skipTables { + if Glob(t, name) { + return false + } + } + for _, t := range tables { + if Glob(t, name) { + return true + } + } + return false +} + +// Glob will test a string pattern, potentially containing globs, against a +// subject string. The result is a simple true/false, determining whether or +// not the glob pattern matched the subject text. +func Glob(pattern, subj string) bool { + // Empty pattern can only match empty subject + if pattern == "" { + return subj == pattern + } + + // If the pattern _is_ a glob, it matches everything + if pattern == GLOB { + return true + } + + parts := strings.Split(pattern, GLOB) + + if len(parts) == 1 { + // No globs in pattern, so test for equality + return subj == pattern + } + + leadingGlob := strings.HasPrefix(pattern, GLOB) + trailingGlob := strings.HasSuffix(pattern, GLOB) + end := len(parts) - 1 + + // Go over the leading parts and ensure they match. + for i := 0; i < end; i++ { + idx := strings.Index(subj, parts[i]) + + switch i { + case 0: + // Check the first section. Requires special handling. + if !leadingGlob && idx != 0 { + return false + } + default: + // Check that the middle parts match. + if idx < 0 { + return false + } + } + + // Trim evaluated text from subj as we loop over the pattern. + subj = subj[idx+len(parts[i]):] + } + + // Reached the last section. Requires special handling. + return trailingGlob || strings.HasSuffix(subj, parts[end]) +} diff --git a/plugin-sdk/glob/glob_test.go b/plugin-sdk/glob/glob_test.go new file mode 100644 index 00000000..fa4edee2 --- /dev/null +++ b/plugin-sdk/glob/glob_test.go @@ -0,0 +1,105 @@ +package glob + +import ( + "strings" + "testing" +) + +func testGlobMatch(t *testing.T, pattern, subj string) { + if !Glob(pattern, subj) { + t.Fatalf("%s should match %s", pattern, subj) + } +} + +func testGlobNoMatch(t *testing.T, pattern, subj string) { + if Glob(pattern, subj) { + t.Fatalf("%s should not match %s", pattern, subj) + } +} + +func TestEmptyPattern(t *testing.T) { + testGlobMatch(t, "", "") + testGlobNoMatch(t, "", "test") +} + +func TestEmptySubject(t *testing.T) { + for _, pattern := range []string{ + "", + "*", + "**", + "***", + "****************", + strings.Repeat("*", 1000000), + } { + testGlobMatch(t, pattern, "") + } + + for _, pattern := range []string{ + // No globs/non-glob characters + "test", + "*test*", + + // Trailing characters + "*x", + "*****************x", + strings.Repeat("*", 1000000) + "x", + + // Leading characters + "x*", + "x*****************", + "x" + strings.Repeat("*", 1000000), + + // Mixed leading/trailing characters + "x*x", + "x****************x", + "x" + strings.Repeat("*", 1000000) + "x", + } { + testGlobNoMatch(t, pattern, "") + } +} + +func TestPatternWithoutGlobs(t *testing.T) { + testGlobMatch(t, "test", "test") +} + +func TestGlob(t *testing.T) { + // Matches + for _, pattern := range []string{ + "*test", // Leading glob + "this*", // Trailing glob + "this*test", // Middle glob + "*is *", // String in between two globs + "*is*a*", // Lots of globs + "**test**", // Double glob characters + "**is**a***test*", // Varying number of globs + "* *", // White space between globs + "*", // Lone glob + "**********", // Nothing but globs + "*Ѿ*", // Unicode with globs + "*is a ϗѾ *", // Mixed ASCII/unicode + } { + testGlobMatch(t, pattern, "this is a ϗѾ test") + } + + // Non-matches + for _, pattern := range []string{ + "test*", // Implicit substring match + "*is", // Partial match + "*no*", // Globs without a match between them + " ", // Plain white space + "* ", // Trailing white space + " *", // Leading white space + "*ʤ*", // Non-matching unicode + "this*this is a test", // Repeated prefix + } { + testGlobNoMatch(t, pattern, "this is a test") + } +} + +func BenchmarkGlob(b *testing.B) { + for i := 0; i < b.N; i++ { + if !Glob("*quick*fox*dog", "The quick brown fox jumped over the lazy dog") { + b.Fatalf("should match") + } + } +} diff --git a/plugin-sdk/helpers/grpczerolog/grpc.go b/plugin-sdk/helpers/grpczerolog/grpc.go new file mode 100644 index 00000000..5d8b35b6 --- /dev/null +++ b/plugin-sdk/helpers/grpczerolog/grpc.go @@ -0,0 +1,29 @@ +package grpczerolog + +import ( + "context" + "fmt" + + "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging" + "github.com/rs/zerolog" +) + +// InterceptorLogger adapts zerolog logger to interceptor logger. +func InterceptorLogger(l zerolog.Logger) logging.Logger { + return logging.LoggerFunc(func(_ context.Context, lvl logging.Level, msg string, fields ...any) { + l := l.With().Fields(fields).Logger() + + switch lvl { + case logging.LevelDebug: + l.Debug().Msg(msg) + case logging.LevelInfo: + l.Info().Msg(msg) + case logging.LevelWarn: + l.Warn().Msg(msg) + case logging.LevelError: + l.Error().Msg(msg) + default: + panic(fmt.Sprintf("unknown level %v", lvl)) + } + }) +} diff --git a/plugin-sdk/helpers/slice.go b/plugin-sdk/helpers/slice.go new file mode 100644 index 00000000..10977d08 --- /dev/null +++ b/plugin-sdk/helpers/slice.go @@ -0,0 +1,36 @@ +package helpers + +import "reflect" + +// InterfaceSlice converts any into a []any slice +func InterfaceSlice(slice any) []any { + // if value is nil return nil + if slice == nil { + return nil + } + s := reflect.ValueOf(slice) + // handle slice behind pointer + if s.Kind() == reflect.Ptr && s.Elem().Kind() == reflect.Slice { + // Keep the distinction between nil and empty slice input + if s.Elem().IsNil() { + return nil + } + + ret := make([]any, s.Elem().Len()) + for i := 0; i < s.Elem().Len(); i++ { + ret[i] = s.Elem().Index(i).Interface() + } + return ret + } + if s.Kind() != reflect.Slice { + return []any{slice} + } + + ret := make([]any, s.Len()) + + for i := 0; i < s.Len(); i++ { + ret[i] = s.Index(i).Interface() + } + + return ret +} diff --git a/plugin-sdk/helpers/slice_test.go b/plugin-sdk/helpers/slice_test.go new file mode 100644 index 00000000..b3b0af68 --- /dev/null +++ b/plugin-sdk/helpers/slice_test.go @@ -0,0 +1,28 @@ +package helpers + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestInterfaceSlice(t *testing.T) { + someStringPtr := "test" + cases := []struct { + Name string + Value any + Want []any + }{ + {Name: "base", Value: []string{"a", "b", "c"}, Want: []any{"a", "b", "c"}}, + {Name: "nil", Value: nil, Want: nil}, + {Name: "empty", Value: []any{}, Want: []any{}}, + {Name: "empty_string_array", Value: []string{}, Want: []any{}}, + {Name: "string_ptr_array", Value: []*string{&someStringPtr}, Want: []any{&someStringPtr}}, + {Name: "string_array_ptr", Value: &[]string{"a"}, Want: []any{"a"}}, + } + for _, tc := range cases { + t.Run(tc.Name, func(t *testing.T) { + assert.Equal(t, tc.Want, InterfaceSlice(tc.Value)) + }) + } +} diff --git a/plugin-sdk/internal/batch/cap.go b/plugin-sdk/internal/batch/cap.go new file mode 100644 index 00000000..58c9f397 --- /dev/null +++ b/plugin-sdk/internal/batch/cap.go @@ -0,0 +1,72 @@ +package batch + +type capped struct { + current, limit int64 +} + +func (c capped) reachedLimit() bool { return c.limit > 0 && c.current >= c.limit } + +func (c capped) remaining() int64 { + if c.limit > 0 { + return c.limit - c.current + } + return -1 +} + +func (c capped) remainingPerN(n int64) int64 { + if c.limit > 0 { + return (c.limit - c.current) / n + } + return -1 +} + +func (c capped) cap() int64 { + if c.limit > 0 { + return c.limit + } + return -1 +} + +func (c capped) capPerN(n int64) int64 { + if c.limit > 0 { + return c.limit / n + } + return -1 +} + +type Cap struct { + bytes, rows capped +} + +func (c *Cap) ReachedLimit() bool { return c.bytes.reachedLimit() || c.rows.reachedLimit() } + +func (c *Cap) Rows() int64 { return c.rows.current } + +func (c *Cap) AddRows(rows int64) { c.rows.current += rows } + +func (c *Cap) AddSlice(record *SlicedRecord) { + c.rows.current += record.NumRows() + c.bytes.current += record.Bytes +} + +func (c *Cap) Reset() { + c.bytes.current = 0 + c.rows.current = 0 +} + +func (c *Cap) add(bytes, rows int64) { + c.bytes.current += bytes + c.rows.current += rows +} + +func (c *Cap) set(bytes, rows int64) { + c.bytes.current = bytes + c.rows.current = rows +} + +func CappedAt(bytes, rows int64) *Cap { + return &Cap{ + bytes: capped{limit: bytes}, + rows: capped{limit: rows}, + } +} diff --git a/plugin-sdk/internal/batch/slice.go b/plugin-sdk/internal/batch/slice.go new file mode 100644 index 00000000..df444be9 --- /dev/null +++ b/plugin-sdk/internal/batch/slice.go @@ -0,0 +1,145 @@ +package batch + +import ( + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/util" +) + +type ( + SlicedRecord struct { + arrow.Record + Bytes int64 // we need this as the util.TotalRecordSize will report the full size even for the sliced record + bytesPerRow int64 + } +) + +func (s *SlicedRecord) split(limit *Cap) (add *SlicedRecord, toFlush []arrow.Record, rest *SlicedRecord) { + if s == nil { + return nil, nil, nil + } + + add = s.getAdd(limit) + if add != nil { + limit.add(add.Bytes, add.NumRows()) + } + + if s.Record == nil { + // all processed + return add, nil, nil + } + + toFlush = s.getToFlush(limit) + if s.Record == nil { + // all processed + return add, toFlush, nil + } + + // set bytes & rows new values + limit.set(s.Bytes, s.NumRows()) + return add, toFlush, s +} + +func (s *SlicedRecord) getAdd(limit *Cap) *SlicedRecord { + rowsByBytes := limit.bytes.remainingPerN(s.bytesPerRow) + rows := limit.rows.remaining() + switch { + case rows < 0: + rows = rowsByBytes + case rows > rowsByBytes && rowsByBytes >= 0: + rows = rowsByBytes + } + + switch { + case rows == 0: + return nil + case rows < 0, rows >= s.NumRows(): + // grab the whole record (either no limits or not overflowing) + res := *s + s.Bytes = 0 + s.Record = nil + return &res + } + + res := SlicedRecord{ + Record: s.NewSlice(0, rows), + Bytes: rows * s.bytesPerRow, + bytesPerRow: s.bytesPerRow, + } + s.Record = s.NewSlice(rows, s.NumRows()) + s.Bytes -= res.Bytes + return &res +} + +func (s *SlicedRecord) getToFlush(limit *Cap) []arrow.Record { + rowsByBytes := limit.bytes.capPerN(s.bytesPerRow) + rows := limit.rows.cap() + switch { + case rows < 0: + rows = rowsByBytes + case rows > rowsByBytes && rowsByBytes >= 0: + rows = rowsByBytes + } + + switch { + case rows == 0: + // not even a single row fits + // we still need to process this, so slice by single row + return s.slice() + case rows < 0: + // as s.Record != nil we know that the limits are there in place & the s.Record.NumRows() > 0 + panic("should never be here") + case rows > s.NumRows(): + // no need to flush anything, as the amount of rows isn't enough to grant this + return nil + } + + flush := make([]arrow.Record, 0, s.NumRows()/rows) + offset := int64(0) + for offset+rows <= s.NumRows() { + flush = append(flush, s.NewSlice(offset, offset+rows)) + offset += rows + } + if offset == s.NumRows() { + // we processed everything for flush + s.Record = nil + s.Bytes = 0 + return flush + } + + // set record to the remainder + s.Record = s.NewSlice(offset, s.NumRows()) + s.Bytes = s.NumRows() * s.bytesPerRow + + return flush +} + +func (s *SlicedRecord) slice() []arrow.Record { + res := make([]arrow.Record, s.NumRows()) + for i := int64(0); i < s.NumRows(); i++ { + res[i] = s.NewSlice(i, i+1) + } + return res +} + +func newSlicedRecord(r arrow.Record) *SlicedRecord { + if r.NumRows() == 0 { + return nil + } + res := SlicedRecord{ + Record: r, + Bytes: util.TotalRecordSize(r), + } + res.bytesPerRow = res.Bytes / r.NumRows() + return &res +} + +// SliceRecord will return the SlicedRecord you can add to the batch given the restrictions provided (if any). +// The meaning of the returned values: +// - `add` is good to be added to the current batch that the caller is assembling +// - `flush` represents sliced arrow.Record that needs own batch to be flushed +// - `remaining` represents the overflow of the batch after `add` & `flush` are processed +// Note that the `limit` provided will not be updated. +func SliceRecord(r arrow.Record, limit *Cap) (add *SlicedRecord, flush []arrow.Record, remaining *SlicedRecord) { + l := *limit // copy value + return newSlicedRecord(r).split(&l) +} diff --git a/plugin-sdk/internal/batch/slice_test.go b/plugin-sdk/internal/batch/slice_test.go new file mode 100644 index 00000000..e3d116df --- /dev/null +++ b/plugin-sdk/internal/batch/slice_test.go @@ -0,0 +1,134 @@ +package batch + +import ( + "fmt" + "math/rand" + "strconv" + "testing" + "time" + + "github.com/apache/arrow/go/v16/arrow/util" + "github.com/stretchr/testify/assert" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +func TestSliceRecord(t *testing.T) { + for run := 0; run < 5; run++ { + rows := rand.Intn(100) + 5 + t.Run(strconv.Itoa(rows), func(t *testing.T) { + t.Parallel() + table := schema.TestTable(fmt.Sprintf("test_%d_rows", rows), schema.TestSourceOptions{}) + tg := schema.NewTestDataGenerator(0) + record := tg.Generate(table, schema.GenTestDataOptions{ + MaxRows: rows, + SourceName: "test", + SyncTime: time.Now(), + }) + + recordRows, recordBytes := record.NumRows(), util.TotalRecordSize(record) + + t.Run("only add", func(t *testing.T) { + limit := CappedAt(0, 0) + add, toFlush, rest := SliceRecord(record, limit) + assert.Equal(t, limit.bytes, capped{}) + assert.Equal(t, limit.rows, capped{}) + assert.NotNil(t, add) + assert.Equal(t, recordRows, add.NumRows()) + assert.Equal(t, recordBytes, add.Bytes) + assert.Empty(t, toFlush) + assert.Nil(t, rest) + }) + + t.Run("only single toFlush", func(t *testing.T) { + limit := &Cap{ + bytes: capped{current: recordBytes, limit: recordBytes}, + rows: capped{current: recordRows, limit: recordRows}, + } + add, toFlush, rest := SliceRecord(record, limit) + assert.Equal(t, limit.bytes, capped{current: recordBytes, limit: recordBytes}) + assert.Equal(t, limit.rows, capped{current: recordRows, limit: recordRows}) + assert.Nil(t, add) + assert.NotEmpty(t, toFlush) + assert.Len(t, toFlush, 1) + r := toFlush[0] + assert.Equal(t, recordRows, r.NumRows()) + assert.Nil(t, rest) + }) + + t.Run("full - by rows", func(t *testing.T) { + limit := &Cap{rows: capped{current: recordRows / 10, limit: recordRows / 5}} + remaining := recordRows + + add, toFlush, rest := SliceRecord(record, limit) + assert.Equal(t, limit.bytes, capped{}) + assert.Equal(t, limit.rows, capped{current: recordRows / 10, limit: recordRows / 5}) + + // if we could add some rows + if (recordRows/5)-(recordRows/10) > 0 { + assert.NotNil(t, add) + assert.LessOrEqual(t, add.NumRows(), recordRows/5) + assert.LessOrEqual(t, add.Bytes, recordBytes/5) + remaining -= add.NumRows() + } else { + assert.Nil(t, add) + } + + assert.NotEmpty(t, toFlush) + assert.GreaterOrEqual(t, len(toFlush), 4) + for _, f := range toFlush { + assert.LessOrEqual(t, f.NumRows(), recordRows/5) + remaining -= f.NumRows() + } + + assert.GreaterOrEqual(t, remaining, int64(0)) + if remaining == 0 { + assert.Nil(t, rest) + return + } + + assert.NotNil(t, rest) + assert.Less(t, remaining, recordRows/5) + assert.Equal(t, remaining, rest.NumRows()) + assert.Less(t, rest.Bytes, recordBytes/5) + }) + + t.Run("full - by bytes", func(t *testing.T) { + limit := &Cap{bytes: capped{current: recordBytes / 10, limit: recordBytes / 5}} + remaining := recordRows + + add, toFlush, rest := SliceRecord(record, limit) + assert.Equal(t, limit.bytes, capped{current: recordBytes / 10, limit: recordBytes / 5}) + assert.Equal(t, limit.rows, capped{}) + + // if we could add some rows + if (recordBytes/5)-(recordBytes/10) >= util.TotalRecordSize(record)/record.NumRows() { + assert.NotNil(t, add) + assert.LessOrEqual(t, add.NumRows(), recordRows/5) + assert.LessOrEqual(t, add.Bytes, recordBytes/5) + remaining -= add.NumRows() + } else { + assert.Nil(t, add) + } + + assert.NotEmpty(t, toFlush) + assert.GreaterOrEqual(t, len(toFlush), 4) + for _, f := range toFlush { + assert.LessOrEqual(t, f.NumRows(), recordRows/5) + remaining -= f.NumRows() + } + + assert.GreaterOrEqual(t, remaining, int64(0)) + if remaining == 0 { + assert.Nil(t, rest) + return + } + + assert.NotNil(t, rest) + assert.Less(t, remaining, recordRows/5) + assert.Equal(t, remaining, rest.NumRows()) + assert.Less(t, rest.Bytes, recordBytes/5) + }) + }) + } +} diff --git a/plugin-sdk/internal/clients/state/v3/state.go b/plugin-sdk/internal/clients/state/v3/state.go new file mode 100644 index 00000000..4e53e0b5 --- /dev/null +++ b/plugin-sdk/internal/clients/state/v3/state.go @@ -0,0 +1,234 @@ +package state + +import ( + "bytes" + "context" + "io" + "sync" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/ipc" + "github.com/apache/arrow/go/v16/arrow/memory" + pb "github.com/cloudquery/plugin-pb-go/pb/plugin/v3" + "google.golang.org/grpc" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +const ( + keyColumn = "key" + valueColumn = "value" + versionColumn = "version" +) + +type Client struct { + client pb.PluginClient + mem map[string]versionedValue + changes map[string]struct{} // changed keys + mutex *sync.RWMutex + schema *arrow.Schema + versionedMode bool + conn *grpc.ClientConn +} + +type versionedValue struct { + value string + version uint64 +} + +func VersionedTable(name string) *schema.Table { + t := table(name) + t.Columns = append(t.Columns, schema.Column{ + // Not defined as PrimaryKey to enable single keys if the destination supports PKs + Name: versionColumn, + Type: arrow.PrimitiveTypes.Uint64, + }) + return t +} + +func NewClient(ctx context.Context, conn *grpc.ClientConn, tableName string) (*Client, error) { + return NewClientWithTable(ctx, conn, table(tableName)) +} + +func NewClientWithTable(ctx context.Context, conn *grpc.ClientConn, table *schema.Table) (*Client, error) { + c := &Client{ + conn: conn, + client: pb.NewPluginClient(conn), + mem: make(map[string]versionedValue), + changes: make(map[string]struct{}), + mutex: &sync.RWMutex{}, + versionedMode: table.Column(versionColumn) != nil, + } + sc := table.ToArrowSchema() + c.schema = sc + tableBytes, err := pb.SchemaToBytes(sc) + if err != nil { + return nil, err + } + + writeClient, err := c.client.Write(ctx) + if err != nil { + return nil, err + } + if err := writeClient.Send(&pb.Write_Request{ + Message: &pb.Write_Request_MigrateTable{ + MigrateTable: &pb.Write_MessageMigrateTable{ + Table: tableBytes, + }, + }, + }); err != nil { + return nil, err + } + if _, err := writeClient.CloseAndRecv(); err != nil { + return nil, err + } + + readClient, err := c.client.Read(ctx, &pb.Read_Request{ + Table: tableBytes, + }) + if err != nil { + return nil, err + } + c.mutex.Lock() + defer c.mutex.Unlock() + for { + res, err := readClient.Recv() + if err != nil { + if err == io.EOF { + break + } + return nil, err + } + rdr, err := ipc.NewReader(bytes.NewReader(res.Record)) + if err != nil { + return nil, err + } + for { + record, err := rdr.Read() + if err != nil { + if err == io.EOF { + break + } + return nil, err + } + if record.NumRows() == 0 { + continue + } + keys := record.Columns()[0].(*array.String) + values := record.Columns()[1].(*array.String) + + var versions *array.Uint64 + if c.versionedMode { + versions = record.Columns()[2].(*array.Uint64) + } + for i := 0; i < keys.Len(); i++ { + k, val := keys.Value(i), values.Value(i) + + var ver uint64 + if versions != nil && versions.IsValid(i) { + ver = versions.Value(i) + } + if cur, ok := c.mem[k]; ok { + if cur.version > ver { + continue + } + } + c.mem[k] = versionedValue{ + value: val, + version: ver, + } + } + } + } + + return c, nil +} + +func (c *Client) SetKey(_ context.Context, key string, value string) error { + c.mutex.Lock() + defer c.mutex.Unlock() + + if c.mem[key].value == value { + return nil // don't update if the value is the same + } + c.mem[key] = versionedValue{ + value: value, + version: c.mem[key].version + 1, + } + c.changes[key] = struct{}{} + return nil +} + +func (c *Client) Flush(ctx context.Context) error { + c.mutex.Lock() + defer c.mutex.Unlock() + bldr := array.NewRecordBuilder(memory.DefaultAllocator, c.schema) + keys, values := bldr.Field(0).(*array.StringBuilder), bldr.Field(1).(*array.StringBuilder) + var version *array.Uint64Builder + if c.versionedMode { + version = bldr.Field(2).(*array.Uint64Builder) + } + for k := range c.changes { + val := c.mem[k] + keys.Append(k) + values.Append(val.value) + if version != nil { + version.Append(val.version) + } + } + rec := bldr.NewRecord() + recordBytes, err := pb.RecordToBytes(rec) + if err != nil { + return err + } + writeClient, err := c.client.Write(ctx) + if err != nil { + return err + } + if err := writeClient.Send(&pb.Write_Request{ + Message: &pb.Write_Request_Insert{ + Insert: &pb.Write_MessageInsert{ + Record: recordBytes, + }, + }, + }); err != nil { + return err + } + if _, err := writeClient.CloseAndRecv(); err != nil { + return err + } + + c.changes = make(map[string]struct{}) + return nil +} + +func (c *Client) GetKey(_ context.Context, key string) (string, error) { + c.mutex.RLock() + defer c.mutex.RUnlock() + return c.mem[key].value, nil +} + +func (c *Client) Close() error { + if c.conn != nil { + return c.conn.Close() + } + return nil +} + +func table(name string) *schema.Table { + return &schema.Table{ + Name: name, + Columns: []schema.Column{ + { + Name: keyColumn, + Type: arrow.BinaryTypes.String, + PrimaryKey: true, + }, + { + Name: valueColumn, + Type: arrow.BinaryTypes.String, + }, + }, + } +} diff --git a/plugin-sdk/internal/clients/state/v3/state_test.go b/plugin-sdk/internal/clients/state/v3/state_test.go new file mode 100644 index 00000000..ab446d55 --- /dev/null +++ b/plugin-sdk/internal/clients/state/v3/state_test.go @@ -0,0 +1,3 @@ +package state + +// Note: State is tested under serve/state_test.go with a real plugin server. diff --git a/plugin-sdk/internal/memdb/memdb.go b/plugin-sdk/internal/memdb/memdb.go new file mode 100644 index 00000000..e8db5b4e --- /dev/null +++ b/plugin-sdk/internal/memdb/memdb.go @@ -0,0 +1,330 @@ +package memdb + +import ( + "context" + "fmt" + "sync" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/rs/zerolog" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/plugin" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" +) + +// client is mostly used for testing the destination plugin. +type client struct { + memoryDB map[string][]arrow.Record + tables map[string]*schema.Table + memoryDBLock sync.RWMutex + errOnWrite bool + blockingWrite bool +} + +type Option func(*client) + +type Spec struct { +} + +func WithErrOnWrite() Option { + return func(c *client) { + c.errOnWrite = true + } +} + +func WithBlockingWrite() Option { + return func(c *client) { + c.blockingWrite = true + } +} + +func GetNewClient(options ...Option) plugin.NewClientFunc { + c := &client{ + memoryDB: make(map[string][]arrow.Record), + memoryDBLock: sync.RWMutex{}, + tables: map[string]*schema.Table{ + "table1": { + Name: "table1", + Columns: []schema.Column{ + { + Name: "col1", + Type: arrow.PrimitiveTypes.Int64, + Description: "col1 description", + PrimaryKey: true, + NotNull: true, + IncrementalKey: false, + Unique: true, + }, + }, + Relations: schema.Tables{ + { + Name: "table2", + Columns: []schema.Column{ + { + Name: "col1", + Type: types.UUID, + Description: "col1 description", + PrimaryKey: false, + NotNull: false, + IncrementalKey: true, + Unique: false, + }, + }, + Relations: schema.Tables{ + { + Name: "table3", + Columns: []schema.Column{ + { + Name: "col1", + Type: types.UUID, + Description: "col1 description", + PrimaryKey: false, + NotNull: false, + IncrementalKey: true, + Unique: false, + }, + }, + IsPaid: true, + }, + }, + }, + }, + }, + }, + } + for _, opt := range options { + opt(c) + } + return func(context.Context, zerolog.Logger, []byte, plugin.NewClientOptions, *aws.Config) (plugin.Client, error) { + return c, nil + } +} + +func NewMemDBClient(ctx context.Context, l zerolog.Logger, spec []byte, options plugin.NewClientOptions, overrideConfig *aws.Config) (plugin.Client, error) { + return GetNewClient()(ctx, l, spec, options, overrideConfig) +} + +func NewMemDBClientErrOnNew(context.Context, zerolog.Logger, []byte, plugin.NewClientOptions, *aws.Config) (plugin.Client, error) { + return nil, fmt.Errorf("newTestDestinationMemDBClientErrOnNew") +} + +func (c *client) overwrite(table *schema.Table, record arrow.Record) { + for i := int64(0); i < record.NumRows(); i++ { + c.overwriteRow(table, record.NewSlice(i, i+1)) + } +} + +func (c *client) overwriteRow(table *schema.Table, data arrow.Record) { + tableName := table.Name + pksIndex := table.PrimaryKeysIndexes() + if len(pksIndex) == 0 { + c.memoryDB[tableName] = append(c.memoryDB[tableName], data) + return + } + + for i, row := range c.memoryDB[tableName] { + found := true + for _, pkIndex := range pksIndex { + s1 := data.Column(pkIndex).String() + s2 := row.Column(pkIndex).String() + if s1 != s2 { + found = false + } + } + if found { + c.memoryDB[tableName] = append(c.memoryDB[tableName][:i], c.memoryDB[tableName][i+1:]...) + c.memoryDB[tableName] = append(c.memoryDB[tableName], data) + return + } + } + c.memoryDB[tableName] = append(c.memoryDB[tableName], data) +} + +func (*client) ID() string { + return "testDestinationMemDB" +} + +func (*client) GetSpec() any { + return &Spec{} +} + +func (c *client) Read(_ context.Context, table *schema.Table, res chan<- arrow.Record) error { + c.memoryDBLock.RLock() + defer c.memoryDBLock.RUnlock() + + tableName := table.Name + // we iterate over records in reverse here because we don't set an expectation + // of ordering on plugins, and we want to make sure that the tests are not + // dependent on the order of insertion either. + rows := c.memoryDB[tableName] + for i := len(rows) - 1; i >= 0; i-- { + res <- rows[i] + } + return nil +} + +func (c *client) Sync(_ context.Context, options plugin.SyncOptions, res chan<- message.SyncMessage, _ chan<- *schema.Resource) error { + c.memoryDBLock.RLock() + + for tableName := range c.memoryDB { + if !plugin.MatchesTable(tableName, options.Tables, options.SkipTables) { + continue + } + for _, row := range c.memoryDB[tableName] { + res <- &message.SyncInsert{ + Record: row, + } + } + } + c.memoryDBLock.RUnlock() + return nil +} + +func (c *client) Tables(_ context.Context, opts plugin.TableOptions) (schema.Tables, error) { + tables := make(schema.Tables, 0, len(c.tables)) + for _, table := range c.tables { + tables = append(tables, table) + } + return tables.FilterDfs(opts.Tables, opts.SkipTables, opts.SkipDependentTables) +} + +func (c *client) migrate(_ context.Context, table *schema.Table) { + tableName := table.Name + memTable := c.memoryDB[tableName] + if memTable == nil { + c.memoryDB[tableName] = make([]arrow.Record, 0) + c.tables[tableName] = table + return + } + + changes := table.GetChanges(c.tables[tableName]) + // memdb doesn't support any auto-migrate + if changes == nil { + return + } + c.memoryDB[tableName] = make([]arrow.Record, 0) + c.tables[tableName] = table +} + +func (c *client) Write(ctx context.Context, msgs <-chan message.WriteMessage) error { + if c.errOnWrite { + return fmt.Errorf("errOnWrite") + } + if c.blockingWrite { + <-ctx.Done() + if c.errOnWrite { + return fmt.Errorf("errOnWrite") + } + return nil + } + + for msg := range msgs { + c.memoryDBLock.Lock() + + switch msg := msg.(type) { + case *message.WriteMigrateTable: + c.migrate(ctx, msg.Table) + case *message.WriteDeleteStale: + c.deleteStale(ctx, msg) + case *message.WriteDeleteRecord: + c.deleteRecord(ctx, msg) + case *message.WriteInsert: + sc := msg.Record.Schema() + tableName, ok := sc.Metadata().GetValue(schema.MetadataTableName) + if !ok { + return fmt.Errorf("table name not found in schema metadata") + } + table := c.tables[tableName] + c.overwrite(table, msg.Record) + } + + c.memoryDBLock.Unlock() + } + return nil +} + +func (c *client) Close(context.Context) error { + c.memoryDB = nil + return nil +} + +func (c *client) deleteStale(_ context.Context, msg *message.WriteDeleteStale) { + var filteredTable []arrow.Record + tableName := msg.TableName + for i, row := range c.memoryDB[tableName] { + sc := row.Schema() + indices := sc.FieldIndices(schema.CqSourceNameColumn.Name) + if len(indices) == 0 { + continue + } + sourceColIndex := indices[0] + indices = sc.FieldIndices(schema.CqSyncTimeColumn.Name) + if len(indices) == 0 { + continue + } + syncColIndex := indices[0] + + if row.Column(sourceColIndex).(*array.String).Value(0) == msg.SourceName { + unit := row.Column(syncColIndex).DataType().(*arrow.TimestampType).Unit + rowSyncTime := row.Column(syncColIndex).(*array.Timestamp).Value(0).ToTime(unit).UTC() + if !rowSyncTime.Before(msg.SyncTime) { + filteredTable = append(filteredTable, c.memoryDB[tableName][i]) + } + } + } + c.memoryDB[tableName] = filteredTable +} + +func (c *client) deleteRecord(_ context.Context, msg *message.WriteDeleteRecord) { + var filteredTable []arrow.Record + tableName := msg.TableName + for i, row := range c.memoryDB[tableName] { + isMatch := true + // Groups are evaluated as AND + for _, predGroup := range msg.WhereClause { + for _, pred := range predGroup.Predicates { + predResult := evaluatePredicate(pred, row) + if predGroup.GroupingType == "AND" { + isMatch = isMatch && predResult + } else if predResult { + isMatch = true + break + } + } + // If any single predicate group is false then we can break out of the loop + if !isMatch { + break + } + } + + if !isMatch { + filteredTable = append(filteredTable, c.memoryDB[tableName][i]) + } + } + c.memoryDB[tableName] = filteredTable +} + +func evaluatePredicate(pred message.Predicate, record arrow.Record) bool { + sc := record.Schema() + indices := sc.FieldIndices(pred.Column) + if len(indices) == 0 { + return false + } + syncColIndex := indices[0] + + if record.Column(syncColIndex).DataType() != pred.Record.Column(0).DataType() { + return false + } + // dataType := record.Column(syncColIndex).DataType() + switch pred.Operator { + case "eq": + return record.Column(syncColIndex).String() == pred.Record.Column(0).String() + // return record.Column(syncColIndex).(*array.String).Value(0) == pred.Record.Column(0).(*array.String).Value(0) + default: + return false + } +} diff --git a/plugin-sdk/internal/memdb/memdb_test.go b/plugin-sdk/internal/memdb/memdb_test.go new file mode 100644 index 00000000..c00576d0 --- /dev/null +++ b/plugin-sdk/internal/memdb/memdb_test.go @@ -0,0 +1,80 @@ +package memdb + +import ( + "context" + "testing" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/plugin" +) + +func TestPlugin(t *testing.T) { + ctx := context.Background() + p := plugin.NewPlugin("test", "development", NewMemDBClient) + if err := p.Init(ctx, nil, plugin.NewClientOptions{}); err != nil { + t.Fatal(err) + } + plugin.TestWriterSuiteRunner( + t, + p, + plugin.WriterTestSuiteTests{ + SafeMigrations: plugin.SafeMigrations{}, + }, + ) +} + +func TestPluginOnNewError(t *testing.T) { + ctx := context.Background() + p := plugin.NewPlugin("test", "development", NewMemDBClientErrOnNew) + err := p.Init(ctx, nil, plugin.NewClientOptions{}) + + if err == nil { + t.Fatal("expected error") + } +} + +func TestOnWriteError(t *testing.T) { + ctx := context.Background() + newClientFunc := GetNewClient(WithErrOnWrite()) + p := plugin.NewPlugin("test", "development", newClientFunc) + if err := p.Init(ctx, nil, plugin.NewClientOptions{}); err != nil { + t.Fatal(err) + } + if err := p.WriteAll(ctx, nil); err.Error() != "errOnWrite" { + t.Fatalf("expected errOnWrite, got %s", err) + } +} + +// func TestOnWriteCtxCancelled(t *testing.T) { +// ctx := context.Background() +// newClientFunc := GetNewClient(WithBlockingWrite()) +// p := plugin.NewPlugin("test", "development", newClientFunc) +// if err := p.Init(ctx, pbPlugin.Spec{ +// WriteSpec: &pbPlugin.WriteSpec{}, +// }); err != nil { +// t.Fatal(err) +// } +// table := schema.TestTable("test", schema.TestSourceOptions{}) +// tables := schema.Tables{ +// table, +// } +// sourceName := "TestDestinationOnWriteError" +// syncTime := time.Now() +// sourceSpec := pbPlugin.Spec{ +// Name: sourceName, +// } +// ch := make(chan arrow.Record, 1) +// ctx, cancel := context.WithTimeout(ctx, 2*time.Second) +// opts := schema.GenTestDataOptions{ +// SourceName: "test", +// SyncTime: time.Now(), +// MaxRows: 1, +// StableUUID: uuid.Nil, +// } +// record := schema.GenTestData(table, opts)[0] +// ch <- record +// defer cancel() +// err := p.Write(ctx, sourceSpec, tables, syncTime, ch) +// if err != nil { +// t.Fatal(err) +// } +// } diff --git a/plugin-sdk/internal/servers/destination/v0/destinations.go b/plugin-sdk/internal/servers/destination/v0/destinations.go new file mode 100644 index 00000000..230b52de --- /dev/null +++ b/plugin-sdk/internal/servers/destination/v0/destinations.go @@ -0,0 +1,265 @@ +package destination + +import ( + "context" + "encoding/json" + "io" + "sync" + + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + pbBase "github.com/cloudquery/plugin-pb-go/pb/base/v0" + pb "github.com/cloudquery/plugin-pb-go/pb/destination/v0" + "github.com/cloudquery/plugin-pb-go/specs" + schemav2 "github.com/cloudquery/plugin-sdk/v2/schema" + "github.com/rs/zerolog" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/plugin" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +type Server struct { + pb.UnimplementedDestinationServer + Plugin *plugin.Plugin + Logger zerolog.Logger + spec specs.Destination +} + +func (*Server) GetProtocolVersion(context.Context, *pbBase.GetProtocolVersion_Request) (*pbBase.GetProtocolVersion_Response, error) { + return &pbBase.GetProtocolVersion_Response{ + Version: 2, + }, nil +} + +func (s *Server) Configure(ctx context.Context, req *pbBase.Configure_Request) (*pbBase.Configure_Response, error) { + var spec specs.Destination + if err := json.Unmarshal(req.Config, &spec); err != nil { + return nil, status.Errorf(codes.InvalidArgument, "failed to unmarshal spec: %v", err) + } + s.spec = spec + pluginSpec, err := json.Marshal(s.spec.Spec) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "failed to marshal spec: %v", err) + } + return &pbBase.Configure_Response{}, s.Plugin.Init(ctx, pluginSpec, plugin.NewClientOptions{}) +} + +func (s *Server) GetName(context.Context, *pbBase.GetName_Request) (*pbBase.GetName_Response, error) { + return &pbBase.GetName_Response{ + Name: s.Plugin.Name(), + }, nil +} + +func (s *Server) GetVersion(context.Context, *pbBase.GetVersion_Request) (*pbBase.GetVersion_Response, error) { + return &pbBase.GetVersion_Response{ + Version: s.Plugin.Version(), + }, nil +} + +func (s *Server) Migrate(ctx context.Context, req *pb.Migrate_Request) (*pb.Migrate_Response, error) { + var tablesV2 schemav2.Tables + if err := json.Unmarshal(req.Tables, &tablesV2); err != nil { + return nil, status.Errorf(codes.InvalidArgument, "failed to unmarshal tables: %v", err) + } + tables := TablesV2ToV3(tablesV2).FlattenTables() + SetDestinationManagedCqColumns(tables) + s.setPKsForTables(tables) + writeCh := make(chan message.WriteMessage) + eg, ctx := errgroup.WithContext(ctx) + eg.Go(func() error { + return s.Plugin.Write(ctx, writeCh) + }) + for _, table := range tables { + writeCh <- &message.WriteMigrateTable{ + Table: table, + MigrateForce: s.spec.MigrateMode == specs.MigrateModeForced, + } + } + close(writeCh) + if err := eg.Wait(); err != nil { + return nil, status.Errorf(codes.Internal, "failed to write: %v", err) + } + return &pb.Migrate_Response{}, nil +} + +func (*Server) Write(pb.Destination_WriteServer) error { + return status.Errorf(codes.Unimplemented, "method Write is deprecated please upgrade client") +} + +// Note the order of operations in this method is important! +// Trying to insert into the `resources` channel before starting the reader goroutine will cause a deadlock. +func (s *Server) Write2(msg pb.Destination_Write2Server) error { + msgs := make(chan message.WriteMessage) + + r, err := msg.Recv() + if err != nil { + if err == io.EOF { + return msg.SendAndClose(&pb.Write2_Response{}) + } + return status.Errorf(codes.Internal, "failed to receive msg: %v", err) + } + var tablesV2 schemav2.Tables + if err := json.Unmarshal(r.Tables, &tablesV2); err != nil { + return status.Errorf(codes.InvalidArgument, "failed to unmarshal tables: %v", err) + } + var sourceSpec specs.Source + if r.SourceSpec == nil { + // this is for backward compatibility + sourceSpec = specs.Source{ + Name: r.Source, + } + } else { + if err := json.Unmarshal(r.SourceSpec, &sourceSpec); err != nil { + return status.Errorf(codes.InvalidArgument, "failed to unmarshal source spec: %v", err) + } + } + tables := TablesV2ToV3(tablesV2).FlattenTables() + syncTime := r.Timestamp.AsTime() + SetDestinationManagedCqColumns(tables) + s.setPKsForTables(tables) + eg, ctx := errgroup.WithContext(msg.Context()) + // sourceName := r.Source + eg.Go(func() error { + return s.Plugin.Write(ctx, msgs) + }) + + for _, table := range tables { + msgs <- &message.WriteMigrateTable{ + Table: table, + MigrateForce: s.spec.MigrateMode == specs.MigrateModeForced, + } + } + + sourceColumn := &schemav2.Text{} + _ = sourceColumn.Set(sourceSpec.Name) + syncTimeColumn := &schemav2.Timestamptz{} + _ = syncTimeColumn.Set(syncTime) + + for { + r, err := msg.Recv() + if err == io.EOF { + close(msgs) + if err := eg.Wait(); err != nil { + return status.Errorf(codes.Internal, "write failed: %v", err) + } + return msg.SendAndClose(&pb.Write2_Response{}) + } + if err != nil { + close(msgs) + if wgErr := eg.Wait(); wgErr != nil { + return status.Errorf(codes.Internal, "failed to receive msg: %v and write failed: %v", err, wgErr) + } + return status.Errorf(codes.Internal, "failed to receive msg: %v", err) + } + + var origResource schemav2.DestinationResource + if err := json.Unmarshal(r.Resource, &origResource); err != nil { + close(msgs) + if wgErr := eg.Wait(); wgErr != nil { + return status.Errorf(codes.InvalidArgument, "failed to unmarshal resource: %v and write failed: %v", err, wgErr) + } + return status.Errorf(codes.InvalidArgument, "failed to unmarshal resource: %v", err) + } + + table := tables.Get(origResource.TableName) + if table == nil { + close(msgs) + if wgErr := eg.Wait(); wgErr != nil { + return status.Errorf(codes.InvalidArgument, "failed to get table: %s and write failed: %v", origResource.TableName, wgErr) + } + return status.Errorf(codes.InvalidArgument, "failed to get table: %s", origResource.TableName) + } + // this is a check to keep backward compatible for sources that are not adding + // source and sync time + if len(origResource.Data) < len(table.Columns) { + origResource.Data = append([]schemav2.CQType{sourceColumn, syncTimeColumn}, origResource.Data...) + } + convertedResource := CQTypesToRecord(memory.DefaultAllocator, []schemav2.CQTypes{origResource.Data}, table.ToArrowSchema()) + msg := &message.WriteInsert{ + Record: convertedResource, + } + + select { + case msgs <- msg: + case <-ctx.Done(): + close(msgs) + if err := eg.Wait(); err != nil { + return status.Errorf(codes.Internal, "Context done: %v and failed to wait for plugin: %v", ctx.Err(), err) + } + return status.Errorf(codes.Internal, "Context done: %v", ctx.Err()) + } + } +} + +func setCQIDAsPrimaryKeysForTables(tables schema.Tables) { + for _, table := range tables { + for i, col := range table.Columns { + table.Columns[i].PrimaryKey = col.Name == schema.CqIDColumn.Name + } + setCQIDAsPrimaryKeysForTables(table.Relations) + } +} + +// Overwrites or adds the CQ columns that are managed by the destination plugins (_cq_sync_time, _cq_source_name). +func SetDestinationManagedCqColumns(tables []*schema.Table) { + for _, table := range tables { + for i := range table.Columns { + if table.Columns[i].Name == schema.CqIDColumn.Name { + table.Columns[i].Unique = true + table.Columns[i].NotNull = true + } + } + table.OverwriteOrAddColumn(&schema.CqSyncTimeColumn) + table.OverwriteOrAddColumn(&schema.CqSourceNameColumn) + SetDestinationManagedCqColumns(table.Relations) + } +} + +func (*Server) GetMetrics(context.Context, *pb.GetDestinationMetrics_Request) (*pb.GetDestinationMetrics_Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetMetrics is deprecated. Please update CLI") +} + +func (s *Server) DeleteStale(ctx context.Context, req *pb.DeleteStale_Request) (*pb.DeleteStale_Response, error) { + var tablesV2 schemav2.Tables + if err := json.Unmarshal(req.Tables, &tablesV2); err != nil { + return nil, status.Errorf(codes.InvalidArgument, "failed to unmarshal tables: %v", err) + } + tables := TablesV2ToV3(tablesV2).FlattenTables() + SetDestinationManagedCqColumns(tables) + + msgs := make(chan message.WriteMessage) + var writeErr error + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + writeErr = s.Plugin.Write(ctx, msgs) + }() + for _, table := range tables { + bldr := array.NewRecordBuilder(memory.DefaultAllocator, table.ToArrowSchema()) + bldr.Field(table.Columns.Index(schema.CqSourceNameColumn.Name)).(*array.StringBuilder).Append(req.Source) + bldr.Field(table.Columns.Index(schema.CqSyncTimeColumn.Name)).(*array.TimestampBuilder).AppendTime(req.Timestamp.AsTime()) + msgs <- &message.WriteDeleteStale{ + TableName: table.Name, + SourceName: req.Source, + SyncTime: req.Timestamp.AsTime(), + } + } + close(msgs) + wg.Wait() + return &pb.DeleteStale_Response{}, writeErr +} + +func (s *Server) setPKsForTables(tables schema.Tables) { + if s.spec.PKMode == specs.PKModeCQID { + setCQIDAsPrimaryKeysForTables(tables) + } +} + +func (s *Server) Close(ctx context.Context, _ *pb.Close_Request) (*pb.Close_Response, error) { + return &pb.Close_Response{}, s.Plugin.Close(ctx) +} diff --git a/plugin-sdk/internal/servers/destination/v0/schemav2tov3.go b/plugin-sdk/internal/servers/destination/v0/schemav2tov3.go new file mode 100644 index 00000000..dfe64f51 --- /dev/null +++ b/plugin-sdk/internal/servers/destination/v0/schemav2tov3.go @@ -0,0 +1,249 @@ +package destination + +import ( + "encoding/json" + "strings" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + schemav2 "github.com/cloudquery/plugin-sdk/v2/schema" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" +) + +func TablesV2ToV3(tables schemav2.Tables) schema.Tables { + res := make(schema.Tables, len(tables)) + for i, t := range tables { + res[i] = TableV2ToV3(t) + } + return res +} + +func TableV2ToV3(table *schemav2.Table) *schema.Table { + newTable := &schema.Table{ + Name: table.Name, + Description: table.Description, + Columns: ColumnsV2ToV3(table.Columns), + IgnoreInTests: table.IgnoreInTests, + IsIncremental: table.IsIncremental, + } + if len(table.Relations) > 0 { + newTable.Relations = TablesV2ToV3(table.Relations) + } + return newTable +} + +func ColumnsV2ToV3(columns []schemav2.Column) []schema.Column { + res := make([]schema.Column, len(columns)) + for i, c := range columns { + res[i] = ColumnV2ToV3(c) + } + return res +} + +func ColumnV2ToV3(column schemav2.Column) schema.Column { + return schema.Column{ + Name: column.Name, + Description: column.Description, + Type: TypeV2ToV3(column.Type), + NotNull: column.CreationOptions.NotNull, + Unique: column.CreationOptions.Unique, + PrimaryKey: column.CreationOptions.PrimaryKey, + IncrementalKey: column.CreationOptions.IncrementalKey, + IgnoreInTests: column.IgnoreInTests, + } +} + +func TypeV2ToV3(dataType schemav2.ValueType) arrow.DataType { + var typ arrow.DataType + + switch dataType { + case schemav2.TypeBool: + return arrow.FixedWidthTypes.Boolean + case schemav2.TypeInt: + return arrow.PrimitiveTypes.Int64 + case schemav2.TypeFloat: + return arrow.PrimitiveTypes.Float64 + case schemav2.TypeUUID: + return types.ExtensionTypes.UUID + case schemav2.TypeString: + return arrow.BinaryTypes.String + case schemav2.TypeByteArray: + return arrow.BinaryTypes.Binary + case schemav2.TypeStringArray: + return arrow.ListOf(arrow.BinaryTypes.String) + case schemav2.TypeIntArray: + return arrow.ListOf(arrow.PrimitiveTypes.Int64) + case schemav2.TypeTimestamp: + return arrow.FixedWidthTypes.Timestamp_us + case schemav2.TypeJSON: + return types.ExtensionTypes.JSON + case schemav2.TypeUUIDArray: + return arrow.ListOf(types.ExtensionTypes.UUID) + case schemav2.TypeInet: + return types.ExtensionTypes.Inet + case schemav2.TypeInetArray: + return arrow.ListOf(types.ExtensionTypes.Inet) + case schemav2.TypeCIDR: + return types.ExtensionTypes.Inet + case schemav2.TypeCIDRArray: + return arrow.ListOf(types.ExtensionTypes.Inet) + case schemav2.TypeMacAddr: + return types.ExtensionTypes.MAC + case schemav2.TypeMacAddrArray: + return arrow.ListOf(types.ExtensionTypes.MAC) + default: + panic("unknown type " + typ.Name()) + } +} + +func CQTypesOneToRecord(mem memory.Allocator, c schemav2.CQTypes, arrowSchema *arrow.Schema) arrow.Record { + return CQTypesToRecord(mem, []schemav2.CQTypes{c}, arrowSchema) +} + +func CQTypesToRecord(mem memory.Allocator, c []schemav2.CQTypes, arrowSchema *arrow.Schema) arrow.Record { + bldr := array.NewRecordBuilder(mem, arrowSchema) + fields := bldr.Fields() + for i := range fields { + for j := range c { + switch c[j][i].Type() { + case schemav2.TypeBool: + if c[j][i].(*schemav2.Bool).Status == schemav2.Present { + bldr.Field(i).(*array.BooleanBuilder).Append(c[j][i].(*schemav2.Bool).Bool) + } else { + bldr.Field(i).(*array.BooleanBuilder).AppendNull() + } + case schemav2.TypeInt: + if c[j][i].(*schemav2.Int8).Status == schemav2.Present { + bldr.Field(i).(*array.Int64Builder).Append(c[j][i].(*schemav2.Int8).Int) + } else { + bldr.Field(i).(*array.Int64Builder).AppendNull() + } + case schemav2.TypeFloat: + if c[j][i].(*schemav2.Float8).Status == schemav2.Present { + bldr.Field(i).(*array.Float64Builder).Append(c[j][i].(*schemav2.Float8).Float) + } else { + bldr.Field(i).(*array.Float64Builder).AppendNull() + } + case schemav2.TypeString: + if c[j][i].(*schemav2.Text).Status == schemav2.Present { + // In the new type system we wont allow null string as they are not valid utf-8 + // https://github.com/apache/arrow/pull/35161#discussion_r1170516104 + bldr.Field(i).(*array.StringBuilder).Append(strings.ReplaceAll(c[j][i].(*schemav2.Text).Str, "\x00", "")) + } else { + bldr.Field(i).(*array.StringBuilder).AppendNull() + } + case schemav2.TypeByteArray: + if c[j][i].(*schemav2.Bytea).Status == schemav2.Present { + bldr.Field(i).(*array.BinaryBuilder).Append(c[j][i].(*schemav2.Bytea).Bytes) + } else { + bldr.Field(i).(*array.BinaryBuilder).AppendNull() + } + case schemav2.TypeStringArray: + if c[j][i].(*schemav2.TextArray).Status == schemav2.Present { + listBldr := bldr.Field(i).(*array.ListBuilder) + listBldr.Append(true) + for _, str := range c[j][i].(*schemav2.TextArray).Elements { + listBldr.ValueBuilder().(*array.StringBuilder).Append(strings.ReplaceAll(str.Str, "\x00", "")) + } + } else { + bldr.Field(i).(*array.ListBuilder).AppendNull() + } + case schemav2.TypeIntArray: + if c[j][i].(*schemav2.Int8Array).Status == schemav2.Present { + listBldr := bldr.Field(i).(*array.ListBuilder) + listBldr.Append(true) + for _, e := range c[j][i].(*schemav2.Int8Array).Elements { + listBldr.ValueBuilder().(*array.Int64Builder).Append(e.Int) + } + } else { + bldr.Field(i).(*array.ListBuilder).AppendNull() + } + case schemav2.TypeTimestamp: + if c[j][i].(*schemav2.Timestamptz).Status == schemav2.Present { + bldr.Field(i).(*array.TimestampBuilder).Append(arrow.Timestamp(c[j][i].(*schemav2.Timestamptz).Time.UnixMicro())) + } else { + bldr.Field(i).(*array.TimestampBuilder).AppendNull() + } + case schemav2.TypeJSON: + if c[j][i].(*schemav2.JSON).Status == schemav2.Present { + var d any + if err := json.Unmarshal(c[j][i].(*schemav2.JSON).Bytes, &d); err != nil { + panic(err) + } + bldr.Field(i).(*types.JSONBuilder).Append(d) + } else { + bldr.Field(i).(*types.JSONBuilder).AppendNull() + } + case schemav2.TypeUUID: + if c[j][i].(*schemav2.UUID).Status == schemav2.Present { + bldr.Field(i).(*types.UUIDBuilder).Append(c[j][i].(*schemav2.UUID).Bytes) + } else { + bldr.Field(i).(*types.UUIDBuilder).AppendNull() + } + case schemav2.TypeUUIDArray: + if c[j][i].(*schemav2.UUIDArray).Status == schemav2.Present { + listBldr := bldr.Field(i).(*array.ListBuilder) + listBldr.Append(true) + for _, e := range c[j][i].(*schemav2.UUIDArray).Elements { + listBldr.ValueBuilder().(*types.UUIDBuilder).Append(e.Bytes) + } + } else { + bldr.Field(i).(*array.ListBuilder).AppendNull() + } + case schemav2.TypeInet: + if c[j][i].(*schemav2.Inet).Status == schemav2.Present { + bldr.Field(i).(*types.InetBuilder).Append(c[j][i].(*schemav2.Inet).IPNet) + } else { + bldr.Field(i).(*types.InetBuilder).AppendNull() + } + case schemav2.TypeInetArray: + if c[j][i].(*schemav2.InetArray).Status == schemav2.Present { + listBldr := bldr.Field(i).(*array.ListBuilder) + listBldr.Append(true) + for _, e := range c[j][i].(*schemav2.InetArray).Elements { + listBldr.ValueBuilder().(*types.InetBuilder).Append(e.IPNet) + } + } else { + bldr.Field(i).(*array.ListBuilder).AppendNull() + } + case schemav2.TypeCIDR: + if c[j][i].(*schemav2.CIDR).Status == schemav2.Present { + bldr.Field(i).(*types.InetBuilder).Append(c[j][i].(*schemav2.CIDR).IPNet) + } else { + bldr.Field(i).(*types.InetBuilder).AppendNull() + } + case schemav2.TypeCIDRArray: + if c[j][i].(*schemav2.CIDRArray).Status == schemav2.Present { + listBldr := bldr.Field(i).(*array.ListBuilder) + listBldr.Append(true) + for _, e := range c[j][i].(*schemav2.CIDRArray).Elements { + listBldr.ValueBuilder().(*types.InetBuilder).Append(e.IPNet) + } + } else { + bldr.Field(i).(*array.ListBuilder).AppendNull() + } + case schemav2.TypeMacAddr: + if c[j][i].(*schemav2.Macaddr).Status == schemav2.Present { + bldr.Field(i).(*types.MACBuilder).Append(c[j][i].(*schemav2.Macaddr).Addr) + } else { + bldr.Field(i).(*types.MACBuilder).AppendNull() + } + case schemav2.TypeMacAddrArray: + if c[j][i].(*schemav2.MacaddrArray).Status == schemav2.Present { + listBldr := bldr.Field(i).(*array.ListBuilder) + listBldr.Append(true) + for _, e := range c[j][i].(*schemav2.MacaddrArray).Elements { + listBldr.ValueBuilder().(*types.MACBuilder).Append(e.Addr) + } + } else { + bldr.Field(i).(*array.ListBuilder).AppendNull() + } + } + } + } + + return bldr.NewRecord() +} diff --git a/plugin-sdk/internal/servers/destination/v1/convert.go b/plugin-sdk/internal/servers/destination/v1/convert.go new file mode 100644 index 00000000..493e3cc9 --- /dev/null +++ b/plugin-sdk/internal/servers/destination/v1/convert.go @@ -0,0 +1,33 @@ +package destination + +import ( + "bytes" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/ipc" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +// Legacy conversion functions to and from Arrow bytes. From plugin v3 onwards +// this responsibility is handled by plugin-pb-go. + +func NewFromBytes(b []byte) (*arrow.Schema, error) { + rdr, err := ipc.NewReader(bytes.NewReader(b)) + if err != nil { + return nil, err + } + return rdr.Schema(), nil +} + +func NewSchemasFromBytes(b [][]byte) (schema.Schemas, error) { + var err error + ret := make([]*arrow.Schema, len(b)) + for i, buf := range b { + ret[i], err = NewFromBytes(buf) + if err != nil { + return nil, err + } + } + return ret, nil +} diff --git a/plugin-sdk/internal/servers/destination/v1/destination_test.go b/plugin-sdk/internal/servers/destination/v1/destination_test.go new file mode 100644 index 00000000..2d67a795 --- /dev/null +++ b/plugin-sdk/internal/servers/destination/v1/destination_test.go @@ -0,0 +1,160 @@ +package destination + +import ( + "context" + "encoding/json" + "io" + "testing" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + pb "github.com/cloudquery/plugin-pb-go/pb/destination/v1" + pbSource "github.com/cloudquery/plugin-pb-go/pb/source/v2" + "github.com/cloudquery/plugin-pb-go/specs" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/internal/memdb" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/plugin" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +func TestGetName(t *testing.T) { + ctx := context.Background() + s := Server{ + Plugin: plugin.NewPlugin("test", "development", memdb.NewMemDBClient), + } + res, err := s.GetName(ctx, &pb.GetName_Request{}) + if err != nil { + t.Fatal(err) + } + if res.Name != "test" { + t.Fatalf("expected test, got %s", res.GetName()) + } +} + +func TestGetVersion(t *testing.T) { + ctx := context.Background() + s := Server{ + Plugin: plugin.NewPlugin("test", "development", memdb.NewMemDBClient), + } + resVersion, err := s.GetVersion(ctx, &pb.GetVersion_Request{}) + if err != nil { + t.Fatal(err) + } + if resVersion.Version != "development" { + t.Fatalf("expected development, got %s", resVersion.GetVersion()) + } +} + +type mockWriteServer struct { + grpc.ServerStream + messages []*pb.Write_Request +} + +func (*mockWriteServer) SendAndClose(*pb.Write_Response) error { + return nil +} + +func (s *mockWriteServer) Recv() (*pb.Write_Request, error) { + if len(s.messages) > 0 { + msg := s.messages[0] + s.messages = s.messages[1:] + return msg, nil + } + return nil, io.EOF +} + +func (*mockWriteServer) SetHeader(metadata.MD) error { + return nil +} + +func (*mockWriteServer) SendHeader(metadata.MD) error { + return nil +} + +func (*mockWriteServer) SetTrailer(metadata.MD) { +} + +func (*mockWriteServer) Context() context.Context { + return context.Background() +} + +func (*mockWriteServer) SendMsg(any) error { + return nil +} + +func (*mockWriteServer) RecvMsg(any) error { + return nil +} + +func TestPluginSync(t *testing.T) { + ctx := context.Background() + s := Server{ + Plugin: plugin.NewPlugin("test", "development", memdb.NewMemDBClient), + } + destinationSpec := specs.Destination{ + Name: "test", + } + destinationSpecBytes, err := json.Marshal(destinationSpec) + if err != nil { + t.Fatal(err) + } + _, err = s.Configure(ctx, &pb.Configure_Request{ + Config: destinationSpecBytes, + }) + if err != nil { + t.Fatal(err) + } + + writeMockServer := &mockWriteServer{} + if err := s.Write(writeMockServer); err != nil { + t.Fatal(err) + } + table := &schema.Table{ + Name: "test", + Columns: []schema.Column{ + { + Name: "test", + Type: arrow.BinaryTypes.String, + }, + }, + } + schemas := schema.Tables{table}.ToArrowSchemas() + schemaBytes, err := pbSource.SchemasToBytes(schemas) + if err != nil { + t.Fatal(err) + } + sc := table.ToArrowSchema() + bldr := array.NewRecordBuilder(memory.DefaultAllocator, sc) + bldr.Field(0).(*array.StringBuilder).Append("test") + record := bldr.NewRecord() + recordBytes, err := pbSource.RecordToBytes(record) + if err != nil { + t.Fatal(err) + } + + sourceSpec := specs.Source{ + Name: "source_test", + } + sourceSpecBytes, err := json.Marshal(sourceSpec) + if err != nil { + t.Fatal(err) + } + + writeMockServer.messages = []*pb.Write_Request{ + { + Tables: schemaBytes, + Resource: recordBytes, + SourceSpec: sourceSpecBytes, + }, + } + if err := s.Write(writeMockServer); err != nil { + t.Fatal(err) + } + + if _, err := s.Close(ctx, &pb.Close_Request{}); err != nil { + t.Fatal(err) + } +} diff --git a/plugin-sdk/internal/servers/destination/v1/destinations.go b/plugin-sdk/internal/servers/destination/v1/destinations.go new file mode 100644 index 00000000..8668ebf2 --- /dev/null +++ b/plugin-sdk/internal/servers/destination/v1/destinations.go @@ -0,0 +1,241 @@ +package destination + +import ( + "bytes" + "context" + "encoding/json" + "io" + "sync" + + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/ipc" + "github.com/apache/arrow/go/v16/arrow/memory" + pb "github.com/cloudquery/plugin-pb-go/pb/destination/v1" + "github.com/cloudquery/plugin-pb-go/specs" + "github.com/rs/zerolog" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/plugin" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +type Server struct { + pb.UnimplementedDestinationServer + Plugin *plugin.Plugin + Logger zerolog.Logger + spec specs.Destination + migrateMode plugin.MigrateMode +} + +func (s *Server) Configure(ctx context.Context, req *pb.Configure_Request) (*pb.Configure_Response, error) { + var spec specs.Destination + if err := json.Unmarshal(req.Config, &spec); err != nil { + return nil, status.Errorf(codes.InvalidArgument, "failed to unmarshal spec: %v", err) + } + s.spec = spec + pluginSpec, err := json.Marshal(s.spec.Spec) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "failed to marshal spec: %v", err) + } + var pluginSpecMap map[string]any + if err := json.Unmarshal(pluginSpec, &pluginSpecMap); err != nil { + return nil, status.Errorf(codes.InvalidArgument, "failed to unmarshal plugin spec: %v", err) + } + // this is for backward compatibility + if s.spec.BatchSize > 0 { + pluginSpecMap["batch_size"] = s.spec.BatchSize + } + if s.spec.BatchSizeBytes > 0 { + pluginSpecMap["batch_size_bytes"] = s.spec.BatchSizeBytes + } + pluginSpec, err = json.Marshal(pluginSpecMap) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "failed to marshal spec: %v", err) + } + return &pb.Configure_Response{}, s.Plugin.Init(ctx, pluginSpec, plugin.NewClientOptions{}) +} + +func (s *Server) GetName(context.Context, *pb.GetName_Request) (*pb.GetName_Response, error) { + return &pb.GetName_Response{ + Name: s.Plugin.Name(), + }, nil +} + +func (s *Server) GetVersion(context.Context, *pb.GetVersion_Request) (*pb.GetVersion_Response, error) { + return &pb.GetVersion_Response{ + Version: s.Plugin.Version(), + }, nil +} + +func (s *Server) Migrate(ctx context.Context, req *pb.Migrate_Request) (*pb.Migrate_Response, error) { + schemas, err := NewSchemasFromBytes(req.Tables) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "failed to create schemas: %v", err) + } + tables, err := schema.NewTablesFromArrowSchemas(schemas) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "failed to create tables: %v", err) + } + s.setPKsForTables(tables) + + writeCh := make(chan message.WriteMessage) + eg, ctx := errgroup.WithContext(ctx) + eg.Go(func() error { + return s.Plugin.Write(ctx, writeCh) + }) + for _, table := range tables { + writeCh <- &message.WriteMigrateTable{ + Table: table, + MigrateForce: s.migrateMode == plugin.MigrateModeForce, + } + } + close(writeCh) + if err := eg.Wait(); err != nil { + return nil, status.Errorf(codes.Internal, "failed to write: %v", err) + } + return &pb.Migrate_Response{}, nil +} + +// Note the order of operations in this method is important! +// Trying to insert into the `resources` channel before starting the reader goroutine will cause a deadlock. +func (s *Server) Write(msg pb.Destination_WriteServer) error { + msgs := make(chan message.WriteMessage) + + r, err := msg.Recv() + if err != nil { + if err == io.EOF { + return msg.SendAndClose(&pb.Write_Response{}) + } + return status.Errorf(codes.Internal, "failed to receive msg: %v", err) + } + + schemas, err := NewSchemasFromBytes(r.Tables) + if err != nil { + return status.Errorf(codes.InvalidArgument, "failed to create schemas: %v", err) + } + tables, err := schema.NewTablesFromArrowSchemas(schemas) + if err != nil { + return status.Errorf(codes.InvalidArgument, "failed to create tables: %v", err) + } + var sourceSpec specs.Source + if r.SourceSpec == nil { + // this is for backward compatibility + sourceSpec = specs.Source{ + Name: r.Source, + } + } else { + if err := json.Unmarshal(r.SourceSpec, &sourceSpec); err != nil { + return status.Errorf(codes.InvalidArgument, "failed to unmarshal source spec: %v", err) + } + } + s.setPKsForTables(tables) + eg, ctx := errgroup.WithContext(msg.Context()) + + eg.Go(func() error { + return s.Plugin.Write(ctx, msgs) + }) + + for { + r, err := msg.Recv() + if err == io.EOF { + close(msgs) + if err := eg.Wait(); err != nil { + return status.Errorf(codes.Internal, "write failed: %v", err) + } + return msg.SendAndClose(&pb.Write_Response{}) + } + if err != nil { + close(msgs) + if wgErr := eg.Wait(); wgErr != nil { + return status.Errorf(codes.Internal, "failed to receive msg: %v and write failed: %v", err, wgErr) + } + return status.Errorf(codes.Internal, "failed to receive msg: %v", err) + } + rdr, err := ipc.NewReader(bytes.NewReader(r.Resource)) + if err != nil { + close(msgs) + if wgErr := eg.Wait(); wgErr != nil { + return status.Errorf(codes.InvalidArgument, "failed to create reader: %v and write failed: %v", err, wgErr) + } + return status.Errorf(codes.InvalidArgument, "failed to create reader: %v", err) + } + for rdr.Next() { + rec := rdr.Record() + rec.Retain() + msg := &message.WriteInsert{ + Record: rec, + } + select { + case msgs <- msg: + case <-ctx.Done(): + close(msgs) + if err := eg.Wait(); err != nil { + return status.Errorf(codes.Internal, "Context done: %v and failed to wait for plugin: %v", ctx.Err(), err) + } + return status.Errorf(codes.Internal, "Context done: %v", ctx.Err()) + } + } + if err := rdr.Err(); err != nil { + return status.Errorf(codes.InvalidArgument, "failed to read resource: %v", err) + } + } +} + +func setCQIDAsPrimaryKeysForTables(tables schema.Tables) { + for _, table := range tables { + for i, col := range table.Columns { + table.Columns[i].PrimaryKey = col.Name == schema.CqIDColumn.Name + } + setCQIDAsPrimaryKeysForTables(table.Relations) + } +} + +func (*Server) GetMetrics(context.Context, *pb.GetDestinationMetrics_Request) (*pb.GetDestinationMetrics_Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetMetrics is deprecated. please upgrade CLI") +} + +func (s *Server) DeleteStale(ctx context.Context, req *pb.DeleteStale_Request) (*pb.DeleteStale_Response, error) { + schemas, err := NewSchemasFromBytes(req.Tables) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "failed to create schemas: %v", err) + } + tables, err := schema.NewTablesFromArrowSchemas(schemas) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "failed to create tables: %v", err) + } + + msgs := make(chan message.WriteMessage) + var writeErr error + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + writeErr = s.Plugin.Write(ctx, msgs) + }() + for _, table := range tables { + bldr := array.NewRecordBuilder(memory.DefaultAllocator, table.ToArrowSchema()) + bldr.Field(table.Columns.Index(schema.CqSourceNameColumn.Name)).(*array.StringBuilder).Append(req.Source) + bldr.Field(table.Columns.Index(schema.CqSyncTimeColumn.Name)).(*array.TimestampBuilder).AppendTime(req.Timestamp.AsTime()) + msgs <- &message.WriteDeleteStale{ + TableName: table.Name, + SourceName: req.Source, + SyncTime: req.Timestamp.AsTime(), + } + } + close(msgs) + wg.Wait() + return &pb.DeleteStale_Response{}, writeErr +} + +func (s *Server) setPKsForTables(tables schema.Tables) { + if s.spec.PKMode == specs.PKModeCQID { + setCQIDAsPrimaryKeysForTables(tables) + } +} + +func (s *Server) Close(ctx context.Context, _ *pb.Close_Request) (*pb.Close_Response, error) { + return &pb.Close_Response{}, s.Plugin.Close(ctx) +} diff --git a/plugin-sdk/internal/servers/discovery/v0/discovery.go b/plugin-sdk/internal/servers/discovery/v0/discovery.go new file mode 100644 index 00000000..0b44a784 --- /dev/null +++ b/plugin-sdk/internal/servers/discovery/v0/discovery.go @@ -0,0 +1,16 @@ +package discovery + +import ( + "context" + + pb "github.com/cloudquery/plugin-pb-go/pb/discovery/v0" +) + +type Server struct { + pb.UnimplementedDiscoveryServer + Versions []string +} + +func (s *Server) GetVersions(context.Context, *pb.GetVersions_Request) (*pb.GetVersions_Response, error) { + return &pb.GetVersions_Response{Versions: s.Versions}, nil +} diff --git a/plugin-sdk/internal/servers/discovery/v0/discovery_test.go b/plugin-sdk/internal/servers/discovery/v0/discovery_test.go new file mode 100644 index 00000000..0eaab884 --- /dev/null +++ b/plugin-sdk/internal/servers/discovery/v0/discovery_test.go @@ -0,0 +1,28 @@ +package discovery + +import ( + "context" + "testing" + + pb "github.com/cloudquery/plugin-pb-go/pb/discovery/v0" +) + +func TestDiscovery(t *testing.T) { + ctx := context.Background() + s := &Server{ + Versions: []string{"1", "2"}, + } + resp, err := s.GetVersions(ctx, &pb.GetVersions_Request{}) + if err != nil { + t.Fatal(err) + } + if len(resp.Versions) != 2 { + t.Fatal("expected 2 versions") + } + if resp.Versions[0] != "1" { + t.Fatal("expected version 1") + } + if resp.Versions[1] != "2" { + t.Fatal("expected version 2") + } +} diff --git a/plugin-sdk/internal/servers/discovery/v1/discovery.go b/plugin-sdk/internal/servers/discovery/v1/discovery.go new file mode 100644 index 00000000..896e8a9c --- /dev/null +++ b/plugin-sdk/internal/servers/discovery/v1/discovery.go @@ -0,0 +1,16 @@ +package discovery + +import ( + "context" + + pb "github.com/cloudquery/plugin-pb-go/pb/discovery/v1" +) + +type Server struct { + pb.UnimplementedDiscoveryServer + Versions []int32 +} + +func (s *Server) GetVersions(context.Context, *pb.GetVersions_Request) (*pb.GetVersions_Response, error) { + return &pb.GetVersions_Response{Versions: s.Versions}, nil +} diff --git a/plugin-sdk/internal/servers/discovery/v1/discovery_test.go b/plugin-sdk/internal/servers/discovery/v1/discovery_test.go new file mode 100644 index 00000000..a54b24c7 --- /dev/null +++ b/plugin-sdk/internal/servers/discovery/v1/discovery_test.go @@ -0,0 +1,28 @@ +package discovery + +import ( + "context" + "testing" + + pb "github.com/cloudquery/plugin-pb-go/pb/discovery/v1" +) + +func TestDiscovery(t *testing.T) { + ctx := context.Background() + s := &Server{ + Versions: []int32{1, 2}, + } + resp, err := s.GetVersions(ctx, &pb.GetVersions_Request{}) + if err != nil { + t.Fatal(err) + } + if len(resp.Versions) != 2 { + t.Fatal("expected 2 versions") + } + if resp.Versions[0] != 1 { + t.Fatal("expected version 1") + } + if resp.Versions[1] != 2 { + t.Fatal("expected version 2") + } +} diff --git a/plugin-sdk/internal/servers/plugin/v3/plugin.go b/plugin-sdk/internal/servers/plugin/v3/plugin.go new file mode 100644 index 00000000..d0bdd7a2 --- /dev/null +++ b/plugin-sdk/internal/servers/plugin/v3/plugin.go @@ -0,0 +1,401 @@ +package plugin + +import ( + "context" + "errors" + "fmt" + "io" + + "github.com/apache/arrow/go/v16/arrow" + pb "github.com/cloudquery/plugin-pb-go/pb/plugin/v3" + "github.com/rs/zerolog" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/trace" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/plugin" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +const MaxMsgSize = 100 * 1024 * 1024 // 100 MiB + +type Server struct { + pb.UnimplementedPluginServer + Plugin *plugin.Plugin + Logger zerolog.Logger + Directory string +} + +func (s *Server) GetTables(ctx context.Context, req *pb.GetTables_Request) (*pb.GetTables_Response, error) { + tables, err := s.Plugin.Tables(ctx, plugin.TableOptions{ + Tables: req.Tables, + SkipTables: req.SkipTables, + SkipDependentTables: req.SkipDependentTables, + }) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to get tables: %v", err) + } + schemas := tables.ToArrowSchemas() + encoded := make([][]byte, len(schemas)) + for i, sc := range schemas { + encoded[i], err = pb.SchemaToBytes(sc) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to encode tables: %v", err) + } + } + return &pb.GetTables_Response{ + Tables: encoded, + }, nil +} + +func (s *Server) GetName(context.Context, *pb.GetName_Request) (*pb.GetName_Response, error) { + return &pb.GetName_Response{ + Name: s.Plugin.Name(), + }, nil +} + +func (s *Server) GetVersion(context.Context, *pb.GetVersion_Request) (*pb.GetVersion_Response, error) { + return &pb.GetVersion_Response{ + Version: s.Plugin.Version(), + }, nil +} + +func (s *Server) GetSpecSchema(context.Context, *pb.GetSpecSchema_Request) (*pb.GetSpecSchema_Response, error) { + sc := s.Plugin.JSONSchema() + if len(sc) == 0 { + return &pb.GetSpecSchema_Response{}, nil + } + + return &pb.GetSpecSchema_Response{JsonSchema: &sc}, nil +} + +func (s *Server) TestConnection(ctx context.Context, req *pb.TestConnection_Request) (*pb.TestConnection_Response, error) { + err := s.Plugin.TestConnection(ctx, s.Logger, req.Spec) + if err == nil { + return &pb.TestConnection_Response{Success: true}, nil + } + + const unknown = "UNKNOWN" + var testConnErr *plugin.TestConnError + if !errors.As(err, &testConnErr) { + if errors.Is(err, plugin.ErrNotImplemented) { + return nil, status.Errorf(codes.Unimplemented, "TestConnection feature is not implemented in this plugin") + } + + return &pb.TestConnection_Response{ + Success: false, + FailureCode: unknown, + FailureDescription: err.Error(), + }, nil + } + + resp := &pb.TestConnection_Response{ + Success: false, + FailureCode: testConnErr.Code, + } + if resp.FailureCode == "" { + resp.FailureCode = unknown + } + if testConnErr.Message != nil { + resp.FailureDescription = testConnErr.Message.Error() + } + return resp, nil +} + +func (s *Server) Init(ctx context.Context, req *pb.Init_Request) (*pb.Init_Response, error) { + if err := s.Plugin.Init(ctx, req.Spec, plugin.NewClientOptions{NoConnection: req.NoConnection, InvocationID: req.InvocationId}); err != nil { + return nil, status.Errorf(codes.Internal, "failed to init plugin: %v", err) + } + return &pb.Init_Response{}, nil +} + +func (s *Server) Read(req *pb.Read_Request, stream pb.Plugin_ReadServer) error { + records := make(chan arrow.Record) + var readErr error + ctx := stream.Context() + + sc, err := pb.NewSchemaFromBytes(req.Table) + if err != nil { + return status.Errorf(codes.InvalidArgument, "failed to create schema from bytes: %v", err) + } + table, err := schema.NewTableFromArrowSchema(sc) + if err != nil { + return status.Errorf(codes.InvalidArgument, "failed to create table from schema: %v", err) + } + go func() { + defer close(records) + err := s.Plugin.Read(ctx, table, records) + if err != nil { + readErr = fmt.Errorf("failed to read records: %w", err) + } + }() + + for rec := range records { + recBytes, err := pb.RecordToBytes(rec) + if err != nil { + return status.Errorf(codes.Internal, "failed to convert record to bytes: %v", err) + } + resp := &pb.Read_Response{ + Record: recBytes, + } + if err := stream.Send(resp); err != nil { + return status.Errorf(codes.Internal, "failed to send read response: %v", err) + } + } + + return readErr +} + +func flushMetrics() { + traceProvider, ok := otel.GetTracerProvider().(*trace.TracerProvider) + if ok && traceProvider != nil { + traceProvider.ForceFlush(context.Background()) + } + meterProvider, ok := otel.GetMeterProvider().(*metric.MeterProvider) + if ok && meterProvider != nil { + meterProvider.ForceFlush(context.Background()) + } +} + +func (s *Server) Sync(req *pb.Sync_Request, stream pb.Plugin_SyncServer) error { + msgs := make(chan message.SyncMessage) + resourceCh := make(chan *schema.Resource) + var syncErr error + ctx := stream.Context() + + syncOptions := plugin.SyncOptions{ + Tables: req.Tables, + SkipTables: req.SkipTables, + SkipDependentTables: req.SkipDependentTables, + DeterministicCQID: req.DeterministicCqId, + } + if req.Backend != nil { + syncOptions.BackendOptions = &plugin.BackendOptions{ + TableName: req.Backend.TableName, + Connection: req.Backend.Connection, + } + } + + go func() { + defer flushMetrics() + defer close(msgs) + err := s.Plugin.Sync(ctx, syncOptions, msgs, resourceCh) + if err != nil { + syncErr = fmt.Errorf("failed to sync records: %w", err) + } + }() + var err error + for msg := range msgs { + msg, err = s.Plugin.OnBeforeSend(ctx, msg) + if err != nil { + syncErr = fmt.Errorf("failed before sending message: %w", err) + return syncErr + } + pbMsg := &pb.Sync_Response{} + switch m := msg.(type) { + case *message.SyncMigrateTable: + tableSchema := m.Table.ToArrowSchema() + schemaBytes, err := pb.SchemaToBytes(tableSchema) + if err != nil { + return status.Errorf(codes.Internal, "failed to encode table schema: %v", err) + } + pbMsg.Message = &pb.Sync_Response_MigrateTable{ + MigrateTable: &pb.Sync_MessageMigrateTable{ + Table: schemaBytes, + }, + } + + case *message.SyncInsert: + recordBytes, err := pb.RecordToBytes(m.Record) + if err != nil { + return status.Errorf(codes.Internal, "failed to encode record: %v", err) + } + pbMsg.Message = &pb.Sync_Response_Insert{ + Insert: &pb.Sync_MessageInsert{ + Record: recordBytes, + }, + } + case *message.SyncDeleteRecord: + whereClause := make([]*pb.PredicatesGroup, len(m.WhereClause)) + for j, predicateGroup := range m.WhereClause { + whereClause[j] = &pb.PredicatesGroup{ + GroupingType: pb.PredicatesGroup_GroupingType(pb.PredicatesGroup_GroupingType_value[predicateGroup.GroupingType]), + Predicates: make([]*pb.Predicate, len(predicateGroup.Predicates)), + } + for i, predicate := range predicateGroup.Predicates { + record, err := pb.RecordToBytes(predicate.Record) + if err != nil { + return status.Errorf(codes.Internal, "failed to encode record: %v", err) + } + + whereClause[j].Predicates[i] = &pb.Predicate{ + Record: record, + Column: predicate.Column, + Operator: pb.Predicate_Operator(pb.Predicate_Operator_value[predicate.Operator]), + } + } + } + + tableRelations := make([]*pb.TableRelation, len(m.TableRelations)) + for i, tr := range m.TableRelations { + tableRelations[i] = &pb.TableRelation{ + TableName: tr.TableName, + ParentTable: tr.ParentTable, + } + } + pbMsg.Message = &pb.Sync_Response_DeleteRecord{ + DeleteRecord: &pb.Sync_MessageDeleteRecord{ + TableName: m.TableName, + TableRelations: tableRelations, + WhereClause: whereClause, + }, + } + default: + return status.Errorf(codes.Internal, "unknown message type: %T", msg) + } + + size := proto.Size(pbMsg) + if size > MaxMsgSize { + s.Logger.Error().Int("bytes", size).Msg("Message exceeds max size") + continue + } + if err := stream.Send(pbMsg); err != nil { + return status.Errorf(codes.Internal, "failed to send message: %v", err) + } + } + + if err := s.Plugin.OnSyncFinish(ctx); err != nil { + return status.Errorf(codes.Internal, "failed to finish sync: %v", err) + } + + return syncErr +} + +func (s *Server) Write(stream pb.Plugin_WriteServer) error { + msgs := make(chan message.WriteMessage) + ctx := stream.Context() + eg, gctx := errgroup.WithContext(ctx) + eg.Go(func() error { + return s.Plugin.Write(gctx, msgs) + }) + + for { + r, err := stream.Recv() + if err == io.EOF { + close(msgs) + if err := eg.Wait(); err != nil { + return status.Errorf(codes.Internal, "write failed: %v", err) + } + return stream.SendAndClose(&pb.Write_Response{}) + } + if err != nil { + close(msgs) + if wgErr := eg.Wait(); wgErr != nil { + return status.Errorf(codes.Internal, "failed to receive msg: %v and write failed: %v", err, wgErr) + } + return status.Errorf(codes.Internal, "failed to receive msg: %v", err) + } + var pluginMessage message.WriteMessage + var pbMsgConvertErr error + switch pbMsg := r.Message.(type) { + case *pb.Write_Request_MigrateTable: + sc, err := pb.NewSchemaFromBytes(pbMsg.MigrateTable.Table) + if err != nil { + pbMsgConvertErr = status.Errorf(codes.InvalidArgument, "failed to create schema from bytes: %v", err) + break + } + table, err := schema.NewTableFromArrowSchema(sc) + if err != nil { + pbMsgConvertErr = status.Errorf(codes.InvalidArgument, "failed to create table from schema: %v", err) + break + } + pluginMessage = &message.WriteMigrateTable{ + Table: table, + MigrateForce: pbMsg.MigrateTable.MigrateForce, + } + case *pb.Write_Request_Insert: + record, err := pb.NewRecordFromBytes(pbMsg.Insert.Record) + if err != nil { + pbMsgConvertErr = status.Errorf(codes.InvalidArgument, "failed to create record: %v", err) + break + } + pluginMessage = &message.WriteInsert{ + Record: record, + } + case *pb.Write_Request_Delete: + pluginMessage = &message.WriteDeleteStale{ + TableName: pbMsg.Delete.TableName, + SourceName: pbMsg.Delete.SourceName, + SyncTime: pbMsg.Delete.SyncTime.AsTime(), + } + + case *pb.Write_Request_DeleteRecord: + whereClause := make(message.PredicateGroups, len(pbMsg.DeleteRecord.WhereClause)) + + for j, predicateGroup := range pbMsg.DeleteRecord.WhereClause { + whereClause[j].GroupingType = predicateGroup.GroupingType.String() + whereClause[j].Predicates = make(message.Predicates, len(predicateGroup.Predicates)) + for i, predicate := range predicateGroup.Predicates { + record, err := pb.NewRecordFromBytes(predicate.Record) + if err != nil { + pbMsgConvertErr = status.Errorf(codes.InvalidArgument, "failed to create record: %v", err) + break + } + whereClause[j].Predicates[i] = message.Predicate{ + Record: record, + Column: predicate.Column, + Operator: predicate.Operator.String(), + } + } + } + + tableRelations := make([]message.TableRelation, len(pbMsg.DeleteRecord.TableRelations)) + for i, tr := range pbMsg.DeleteRecord.TableRelations { + tableRelations[i] = message.TableRelation{ + TableName: tr.TableName, + ParentTable: tr.ParentTable, + } + } + pluginMessage = &message.WriteDeleteRecord{ + DeleteRecord: message.DeleteRecord{ + TableName: pbMsg.DeleteRecord.TableName, + TableRelations: tableRelations, + WhereClause: whereClause, + }, + } + } + + if pbMsgConvertErr != nil { + close(msgs) + if wgErr := eg.Wait(); wgErr != nil { + return status.Errorf(codes.Internal, "failed to convert message: %v and write failed: %v", pbMsgConvertErr, wgErr) + } + return pbMsgConvertErr + } + + select { + case msgs <- pluginMessage: + case <-gctx.Done(): + close(msgs) + if err := eg.Wait(); err != nil { + return status.Errorf(codes.Canceled, "plugin returned error: %v", err) + } + return status.Errorf(codes.Internal, "write failed for unknown reason") + case <-ctx.Done(): + close(msgs) + if err := eg.Wait(); err != nil { + return status.Errorf(codes.Internal, "context done: %v and failed to wait for plugin: %v", ctx.Err(), err) + } + return status.Errorf(codes.Canceled, "context done: %v", ctx.Err()) + } + } +} + +func (s *Server) Close(ctx context.Context, _ *pb.Close_Request) (*pb.Close_Response, error) { + return &pb.Close_Response{}, s.Plugin.Close(ctx) +} diff --git a/plugin-sdk/internal/servers/plugin/v3/plugin_test.go b/plugin-sdk/internal/servers/plugin/v3/plugin_test.go new file mode 100644 index 00000000..2b4b94f2 --- /dev/null +++ b/plugin-sdk/internal/servers/plugin/v3/plugin_test.go @@ -0,0 +1,198 @@ +package plugin + +import ( + "context" + "io" + "testing" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + pb "github.com/cloudquery/plugin-pb-go/pb/plugin/v3" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/internal/memdb" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/plugin" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +func TestGetName(t *testing.T) { + ctx := context.Background() + s := Server{ + Plugin: plugin.NewPlugin("test", "development", memdb.NewMemDBClient), + } + res, err := s.GetName(ctx, &pb.GetName_Request{}) + if err != nil { + t.Fatal(err) + } + if res.Name != "test" { + t.Fatalf("expected test, got %s", res.GetName()) + } +} + +func TestGetVersion(t *testing.T) { + ctx := context.Background() + s := Server{ + Plugin: plugin.NewPlugin("test", "development", memdb.NewMemDBClient), + } + resVersion, err := s.GetVersion(ctx, &pb.GetVersion_Request{}) + if err != nil { + t.Fatal(err) + } + if resVersion.Version != "development" { + t.Fatalf("expected development, got %s", resVersion.GetVersion()) + } +} + +type mockSyncServer struct { + grpc.ServerStream + messages []*pb.Sync_Response +} + +func (s *mockSyncServer) Send(*pb.Sync_Response) error { + s.messages = append(s.messages, &pb.Sync_Response{}) + return nil +} + +func (*mockSyncServer) SetHeader(metadata.MD) error { + return nil +} + +func (*mockSyncServer) SendHeader(metadata.MD) error { + return nil +} + +func (*mockSyncServer) SetTrailer(metadata.MD) { +} + +func (*mockSyncServer) Context() context.Context { + return context.Background() +} + +func (*mockSyncServer) SendMsg(any) error { + return nil +} + +func (*mockSyncServer) RecvMsg(any) error { + return nil +} + +type mockWriteServer struct { + grpc.ServerStream + messages []*pb.Write_Request +} + +func (*mockWriteServer) SendAndClose(*pb.Write_Response) error { + return nil +} + +func (s *mockWriteServer) Recv() (*pb.Write_Request, error) { + if len(s.messages) > 0 { + msg := s.messages[0] + s.messages = s.messages[1:] + return msg, nil + } + return nil, io.EOF +} + +func (*mockWriteServer) SetHeader(metadata.MD) error { + return nil +} + +func (*mockWriteServer) SendHeader(metadata.MD) error { + return nil +} + +func (*mockWriteServer) SetTrailer(metadata.MD) { +} + +func (*mockWriteServer) Context() context.Context { + return context.Background() +} + +func (*mockWriteServer) SendMsg(any) error { + return nil +} + +func (*mockWriteServer) RecvMsg(any) error { + return nil +} + +func TestPluginSync(t *testing.T) { + ctx := context.Background() + s := Server{ + Plugin: plugin.NewPlugin("test", "development", memdb.NewMemDBClient), + } + + _, err := s.Init(ctx, &pb.Init_Request{}) + if err != nil { + t.Fatal(err) + } + + streamSyncServer := &mockSyncServer{} + if err := s.Sync(&pb.Sync_Request{}, streamSyncServer); err != nil { + t.Fatal(err) + } + if len(streamSyncServer.messages) != 0 { + t.Fatalf("expected 0 messages, got %d", len(streamSyncServer.messages)) + } + writeMockServer := &mockWriteServer{} + + table := &schema.Table{ + Name: "test", + Columns: []schema.Column{ + { + Name: "test", + Type: arrow.BinaryTypes.String, + }, + }, + } + sc := table.ToArrowSchema() + b, err := pb.SchemaToBytes(sc) + if err != nil { + t.Fatal(err) + } + bldr := array.NewRecordBuilder(memory.DefaultAllocator, sc) + bldr.Field(0).(*array.StringBuilder).Append("test") + record := bldr.NewRecord() + recordBytes, err := pb.RecordToBytes(record) + if err != nil { + t.Fatal(err) + } + + writeMockServer.messages = []*pb.Write_Request{ + { + Message: &pb.Write_Request_MigrateTable{ + MigrateTable: &pb.Write_MessageMigrateTable{ + Table: b, + }, + }, + }, + { + Message: &pb.Write_Request_Insert{ + Insert: &pb.Write_MessageInsert{ + Record: recordBytes, + }, + }, + }, + } + + if err := s.Write(writeMockServer); err != nil { + t.Fatal(err) + } + + streamSyncServer = &mockSyncServer{} + if err := s.Sync(&pb.Sync_Request{ + Tables: []string{"*"}, + }, streamSyncServer); err != nil { + t.Fatal(err) + } + if len(streamSyncServer.messages) != 1 { + t.Fatalf("expected 1 message, got %d", len(streamSyncServer.messages)) + } + + if _, err := s.Close(ctx, &pb.Close_Request{}); err != nil { + t.Fatal(err) + } +} diff --git a/plugin-sdk/message/sync_message.go b/plugin-sdk/message/sync_message.go new file mode 100644 index 00000000..6d6cea50 --- /dev/null +++ b/plugin-sdk/message/sync_message.go @@ -0,0 +1,124 @@ +package message + +import ( + "slices" + + "github.com/apache/arrow/go/v16/arrow" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +type syncBaseMessage struct { +} + +func (*syncBaseMessage) IsSyncMessage() bool { + return true +} + +type SyncMessage interface { + GetTable() *schema.Table + IsSyncMessage() bool +} + +type SyncMigrateTable struct { + syncBaseMessage + Table *schema.Table +} + +func (m SyncMigrateTable) GetTable() *schema.Table { + return m.Table +} + +type SyncInsert struct { + syncBaseMessage + Record arrow.Record +} + +func (m *SyncInsert) GetTable() *schema.Table { + table, err := schema.NewTableFromArrowSchema(m.Record.Schema()) + if err != nil { + panic(err) + } + return table +} + +type SyncMessages []SyncMessage + +type SyncMigrateTables []*SyncMigrateTable + +type SyncInserts []*SyncInsert + +func (messages SyncMessages) InsertItems() int64 { + items := int64(0) + for _, msg := range messages { + if m, ok := msg.(*SyncInsert); ok { + items += m.Record.NumRows() + } + } + return items +} + +func (messages SyncMessages) GetInserts() SyncInserts { + inserts := make(SyncInserts, 0, len(messages)) + for _, msg := range messages { + if m, ok := msg.(*SyncInsert); ok { + inserts = append(inserts, m) + } + } + return slices.Clip(inserts) +} + +func (m SyncMigrateTables) Exists(tableName string) bool { + for _, table := range m { + if table.Table.Name == tableName { + return true + } + } + return false +} + +func (m SyncInserts) Exists(tableName string) bool { + for _, insert := range m { + md := insert.Record.Schema().Metadata() + tableNameMeta, ok := md.GetValue(schema.MetadataTableName) + if !ok { + continue + } + if tableNameMeta == tableName { + return true + } + } + return false +} + +func (m SyncInserts) GetRecords() []arrow.Record { + res := make([]arrow.Record, len(m)) + for i := range m { + res[i] = m[i].Record + } + return res +} + +// Get all records for a single table +func (m SyncInserts) GetRecordsForTable(table *schema.Table) []arrow.Record { + res := make([]arrow.Record, 0, len(m)) + for _, insert := range m { + md := insert.Record.Schema().Metadata() + tableNameMeta, ok := md.GetValue(schema.MetadataTableName) + if !ok || tableNameMeta != table.Name { + continue + } + res = append(res, insert.Record) + } + return slices.Clip(res) +} + +type SyncDeleteRecord struct { + syncBaseMessage + // TODO: Instead of using this struct we should derive the DeletionKeys and parent/child relation from the schema.Table itself + DeleteRecord +} + +func (m SyncDeleteRecord) GetTable() *schema.Table { + return &schema.Table{Name: m.TableName} +} diff --git a/plugin-sdk/message/write_message.go b/plugin-sdk/message/write_message.go new file mode 100644 index 00000000..39dad8a2 --- /dev/null +++ b/plugin-sdk/message/write_message.go @@ -0,0 +1,172 @@ +package message + +import ( + "slices" + "time" + + "github.com/apache/arrow/go/v16/arrow" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +type writeBaseMessage struct{} + +func (*writeBaseMessage) IsWriteMessage() bool { return true } + +type WriteMessage interface { + GetTable() *schema.Table + IsWriteMessage() bool +} + +type WriteMessages []WriteMessage + +func (messages WriteMessages) InsertItems() int64 { + items := int64(0) + for _, msg := range messages { + if m, ok := msg.(*WriteInsert); ok { + items += m.Record.NumRows() + } + } + return items +} + +func (messages WriteMessages) GetInserts() WriteInserts { + inserts := make(WriteInserts, 0, len(messages)) + for _, msg := range messages { + if m, ok := msg.(*WriteInsert); ok { + inserts = append(inserts, m) + } + } + return slices.Clip(inserts) +} + +type WriteMigrateTable struct { + writeBaseMessage + Table *schema.Table + MigrateForce bool +} + +func (m WriteMigrateTable) GetTable() *schema.Table { return m.Table } + +type WriteMigrateTables []*WriteMigrateTable + +func (m WriteMigrateTables) Exists(tableName string) bool { + return slices.ContainsFunc(m, func(msg *WriteMigrateTable) bool { + return msg.Table.Name == tableName + }) +} + +func (m WriteMigrateTables) GetMessageByTable(tableName string) *WriteMigrateTable { + for _, msg := range m { + if msg.Table.Name == tableName { + return msg + } + } + return nil +} + +type WriteInsert struct { + writeBaseMessage + Record arrow.Record +} + +func (m *WriteInsert) GetTable() *schema.Table { + table, err := schema.NewTableFromArrowSchema(m.Record.Schema()) + if err != nil { + panic(err) + } + return table +} + +type WriteInserts []*WriteInsert + +func (m WriteInserts) Exists(tableName string) bool { + return slices.ContainsFunc(m, func(msg *WriteInsert) bool { + tableNameMeta, ok := msg.Record.Schema().Metadata().GetValue(schema.MetadataTableName) + return ok && tableNameMeta == tableName + }) +} + +func (m WriteInserts) GetRecords() []arrow.Record { + res := make([]arrow.Record, len(m)) + for i := range m { + res[i] = m[i].Record + } + return res +} + +func (m WriteInserts) GetRecordsForTable(table *schema.Table) []arrow.Record { + res := make([]arrow.Record, 0, len(m)) + for _, insert := range m { + tableNameMeta, ok := insert.Record.Schema().Metadata().GetValue(schema.MetadataTableName) + if !ok || tableNameMeta != table.Name { + continue + } + res = append(res, insert.Record) + } + return slices.Clip(res) +} + +// WriteDeleteStale is a pretty specific message which requires the destination to be aware of a CLI use-case +// thus it might be deprecated in the future +// in favour of MessageDelete or MessageRawQuery +// The message indicates that the destination needs to run something like "DELETE FROM table WHERE _cq_source_name=$1 and sync_time < $2" +type WriteDeleteStale struct { + writeBaseMessage + TableName string + SourceName string + SyncTime time.Time +} + +func (m WriteDeleteStale) GetTable() *schema.Table { + return &schema.Table{Name: m.TableName} +} + +type WriteDeleteStales []*WriteDeleteStale + +func (m WriteDeleteStales) Exists(tableName string) bool { + return slices.ContainsFunc(m, func(msg *WriteDeleteStale) bool { + return msg.TableName == tableName + }) +} + +type TableRelation struct { + TableName string + ParentTable string +} + +type TableRelations []TableRelation + +type Predicate struct { + Operator string + Column string + Record arrow.Record +} + +type Predicates []Predicate + +type PredicateGroup struct { + // This will be AND or OR + GroupingType string + Predicates Predicates +} + +type PredicateGroups []PredicateGroup + +type DeleteRecord struct { + TableName string + TableRelations TableRelations + WhereClause PredicateGroups + SyncTime time.Time +} + +type WriteDeleteRecord struct { + writeBaseMessage + DeleteRecord +} + +func (m WriteDeleteRecord) GetTable() *schema.Table { + return &schema.Table{Name: m.TableName} +} + +type WriteDeleteRecords []*WriteDeleteRecord diff --git a/plugin-sdk/plugin/diff.go b/plugin-sdk/plugin/diff.go new file mode 100644 index 00000000..47ddcedd --- /dev/null +++ b/plugin-sdk/plugin/diff.go @@ -0,0 +1,51 @@ +package plugin + +import ( + "fmt" + "strings" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" +) + +func RecordsDiff(sc *arrow.Schema, have, want []arrow.Record) string { + return TableDiff(array.NewTableFromRecords(sc, have), array.NewTableFromRecords(sc, want)) +} + +func TableDiff(have, want arrow.Table) string { + if array.TableApproxEqual(have, want, array.WithUnorderedMapKeys(true)) { + return "" + } + + if have.NumCols() != want.NumCols() { + return fmt.Sprintf("different number of columns: %d vs %d", have.NumCols(), want.NumCols()) + } + if have.NumRows() != want.NumRows() { + return fmt.Sprintf("different number of rows: %d vs %d", have.NumRows(), want.NumRows()) + } + + var sb strings.Builder + for i := 0; i < int(have.NumCols()); i++ { + haveCol, err := array.Concatenate(have.Column(i).Data().Chunks(), memory.DefaultAllocator) + if err != nil { + panic(fmt.Errorf("failed to concat left columns at idx %d: %w", i, err)) + } + wantCol, err := array.Concatenate(want.Column(i).Data().Chunks(), memory.DefaultAllocator) + if err != nil { + panic(fmt.Errorf("failed to concat right columns at idx %d: %w", i, err)) + } + edits, err := array.Diff(wantCol, haveCol) + if err != nil { + panic(fmt.Errorf("want: %v, have: %v, error: %w", wantCol.DataType(), haveCol.DataType(), err)) + } + diff := edits.UnifiedDiff(wantCol, haveCol) + if diff != "" { + sb.WriteString(have.Schema().Field(i).Name) + sb.WriteString(": ") + sb.WriteString(diff) + sb.WriteString("\n") + } + } + return sb.String() +} diff --git a/plugin-sdk/plugin/meta.go b/plugin-sdk/plugin/meta.go new file mode 100644 index 00000000..e609a13d --- /dev/null +++ b/plugin-sdk/plugin/meta.go @@ -0,0 +1,10 @@ +package plugin + +import cqapi "github.com/cloudquery/cloudquery-api-go" + +type Meta struct { + Team cqapi.PluginTeam + Kind cqapi.PluginKind + Name cqapi.PluginName + SkipUsageClient bool +} diff --git a/plugin-sdk/plugin/nulls.go b/plugin-sdk/plugin/nulls.go new file mode 100644 index 00000000..95ce9f4c --- /dev/null +++ b/plugin-sdk/plugin/nulls.go @@ -0,0 +1,108 @@ +package plugin + +import ( + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" +) + +func stripNullsFromLists(list array.ListLike) array.ListLike { + // TODO: handle Arrow maps separately if required + bldr := array.NewBuilder(memory.DefaultAllocator, list.DataType()).(array.ListLikeBuilder) + for j := 0; j < list.Len(); j++ { + if list.IsNull(j) { + bldr.AppendNull() + continue + } + bldr.Append(true) + vBldr := bldr.ValueBuilder() + from, to := list.ValueOffsets(j) + slc := array.NewSlice(list.ListValues(), from, to) + for k := 0; k < int(to-from); k++ { + if slc.IsNull(k) { + continue + } + err := vBldr.AppendValueFromString(slc.ValueStr(k)) + if err != nil { + panic(err) + } + } + } + + return bldr.NewArray().(array.ListLike) +} + +type AllowNullFunc func(arrow.DataType) bool + +func (s *WriterTestSuite) replaceNullsByEmpty(arr arrow.Array) arrow.Array { + if s.allowNull == nil { + return arr + } + + if !s.allowNull(arr.DataType()) && arr.NullN() > 0 { + builder := array.NewBuilder(memory.DefaultAllocator, arr.DataType()) + for j := 0; j < arr.Len(); j++ { + if arr.IsNull(j) { + builder.AppendEmptyValue() + continue + } + + if err := builder.AppendValueFromString(arr.ValueStr(j)); err != nil { + panic(err) + } + } + + arr = builder.NewArray() + } + + // we need to process the nested arrays, too + return s.replaceNullsByEmptyNestedArray(arr) +} + +func (s *WriterTestSuite) replaceNullsByEmptyNestedArray(arr arrow.Array) arrow.Array { + if s.allowNull == nil { + return arr + } + + switch arr := arr.(type) { + case array.ListLike: // TODO: handle Arrow maps separately if required + values := s.handleNullsArray(arr.ListValues()) + return array.MakeFromData( + array.NewData(arr.DataType(), arr.Len(), + arr.Data().Buffers(), + []arrow.ArrayData{values.Data()}, + arr.NullN(), arr.Data().Offset(), + ), + ) + case *array.Struct: + children := make([]arrow.ArrayData, arr.NumField()) + for i := 0; i < arr.NumField(); i++ { + children[i] = s.handleNullsArray(arr.Field(i)).Data() + } + return array.MakeFromData( + array.NewData(arr.DataType(), arr.Len(), + arr.Data().Buffers(), + children, + arr.NullN(), arr.Data().Offset(), + ), + ) + default: + return arr + } +} + +func (s *WriterTestSuite) handleNulls(record arrow.Record) arrow.Record { + cols := record.Columns() + for c, col := range cols { + cols[c] = s.handleNullsArray(col) + } + return array.NewRecord(record.Schema(), cols, record.NumRows()) +} + +func (s *WriterTestSuite) handleNullsArray(arr arrow.Array) arrow.Array { + if list, ok := arr.(array.ListLike); ok && s.ignoreNullsInLists { + arr = stripNullsFromLists(list) // TODO: handle Arrow maps separately if required + } + + return s.replaceNullsByEmpty(arr) +} diff --git a/plugin-sdk/plugin/nulls_test.go b/plugin-sdk/plugin/nulls_test.go new file mode 100644 index 00000000..52d26bed --- /dev/null +++ b/plugin-sdk/plugin/nulls_test.go @@ -0,0 +1,102 @@ +package plugin + +import ( + "testing" + "time" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +func TestWithTestIgnoreNullsInLists(t *testing.T) { + s := &WriterTestSuite{ignoreNullsInLists: true} + + tg := schema.NewTestDataGenerator(0) + source := schema.TestTable("ignore_nulls_in_lists", schema.TestSourceOptions{}) + resource := s.handleNulls(tg.Generate(source, schema.GenTestDataOptions{ + SourceName: "allow_null", + SyncTime: time.Now(), + MaxRows: 100, + NullRows: false, + })) + for _, c := range resource.Columns() { + assertNoNullsInLists(t, c) + } + + resource = s.handleNulls(tg.Generate(source, schema.GenTestDataOptions{ + SourceName: "ignore_nulls_in_lists", + SyncTime: time.Now(), + MaxRows: 100, + NullRows: true, + })) + for _, c := range resource.Columns() { + assertNoNullsInLists(t, c) + } +} + +func assertNoNullsInLists(t *testing.T, arr arrow.Array) { + // traverse + switch arr := arr.(type) { + case array.ListLike: + assert.Zero(t, arr.ListValues().NullN()) + case *array.Struct: + for i := 0; i < arr.NumField(); i++ { + assertNoNullsInLists(t, arr.Field(i)) + } + } +} + +func TestWithTestSourceAllowNull(t *testing.T) { + s := &WriterTestSuite{allowNull: func(dt arrow.DataType) bool { + switch dt.(type) { + case *arrow.StructType, arrow.ListLikeType: + return false + default: + return true + } + }} + + tg := schema.NewTestDataGenerator(0) + source := schema.TestTable("allow_null", schema.TestSourceOptions{}) + resource := s.handleNulls(tg.Generate(source, schema.GenTestDataOptions{ + SourceName: "allow_null", + SyncTime: time.Now(), + MaxRows: 100, + NullRows: false, + })) + for _, c := range resource.Columns() { + assertNoNulls(t, s.allowNull, c) + } + + resource = s.handleNulls(tg.Generate(source, schema.GenTestDataOptions{ + SourceName: "allow_null", + SyncTime: time.Now(), + MaxRows: 100, + NullRows: true, + })) + for _, c := range resource.Columns() { + assertNoNulls(t, s.allowNull, c) + } +} + +func assertNoNulls(t *testing.T, allowNull AllowNullFunc, arr arrow.Array) { + require.NotNil(t, allowNull) + + if !allowNull(arr.DataType()) { + assert.Zero(t, arr.NullN()) + } + + // traverse + switch arr := arr.(type) { + case array.ListLike: + assertNoNulls(t, allowNull, arr.ListValues()) + case *array.Struct: + for i := 0; i < arr.NumField(); i++ { + assertNoNulls(t, allowNull, arr.Field(i)) + } + } +} diff --git a/plugin-sdk/plugin/options.go b/plugin-sdk/plugin/options.go new file mode 100644 index 00000000..9de10d86 --- /dev/null +++ b/plugin-sdk/plugin/options.go @@ -0,0 +1,68 @@ +package plugin + +import "github.com/aws/aws-sdk-go-v2/aws" + +type MigrateMode int + +const ( + MigrateModeSafe MigrateMode = iota + MigrateModeForce +) + +var ( + migrateModeStrings = []string{"safe", "force"} +) + +func (m MigrateMode) String() string { + return migrateModeStrings[m] +} + +type Option func(*Plugin) + +func WithBuildTargets(targets []BuildTarget) Option { + return func(p *Plugin) { + p.targets = targets + } +} + +func WithJSONSchema(schema string) Option { + return func(p *Plugin) { + p.schema = schema + } +} + +func WithKind(kind string) Option { + k := Kind(kind) + err := k.Validate() + if err != nil { + panic(err) + } + return func(p *Plugin) { + p.kind = k + } +} + +func WithTeam(team string) Option { + return func(p *Plugin) { + p.team = team + } +} + +// WithConnectionTester can be specified by a plugin to enable explicit connection testing, given a spec. +func WithConnectionTester(tester ConnectionTester) Option { + return func(p *Plugin) { + p.testConnFn = tester + } +} + +func WithCustomConfig(cfg *aws.Config) Option { + return func(p *Plugin) { + p.overrideConfig = cfg + } +} + +type TableOptions struct { + Tables []string + SkipTables []string + SkipDependentTables bool +} diff --git a/plugin-sdk/plugin/plugin.go b/plugin-sdk/plugin/plugin.go new file mode 100644 index 00000000..2ba55e32 --- /dev/null +++ b/plugin-sdk/plugin/plugin.go @@ -0,0 +1,257 @@ +package plugin + +import ( + "context" + "fmt" + "sync" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/aws/aws-sdk-go-v2/aws" + cqapi "github.com/cloudquery/cloudquery-api-go" + "github.com/rs/zerolog" + "github.com/santhosh-tekuri/jsonschema/v5" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +var ErrNotImplemented = fmt.Errorf("not implemented") + +type NewClientOptions struct { + NoConnection bool + InvocationID string + PluginMeta Meta +} + +type NewClientFunc func(context.Context, zerolog.Logger, []byte, NewClientOptions, *aws.Config) (Client, error) + +type Client interface { + SourceClient + DestinationClient +} + +type UnimplementedDestination struct{} + +func (UnimplementedDestination) Write(context.Context, <-chan message.WriteMessage) error { + return ErrNotImplemented +} + +func (UnimplementedDestination) Read(context.Context, *schema.Table, chan<- arrow.Record) error { + return ErrNotImplemented +} + +type UnimplementedSource struct{} + +func (UnimplementedSource) Sync(context.Context, SyncOptions, chan<- message.SyncMessage, chan<- *schema.Resource) error { + return ErrNotImplemented +} + +func (UnimplementedSource) Tables(context.Context, TableOptions) (schema.Tables, error) { + return nil, ErrNotImplemented +} + +// Plugin is the base structure required to pass to sdk.serve +// We take a declarative approach to API here similar to Cobra +type Plugin struct { + // Name of plugin, e.g. aws, gcp, azure etc + name string + // Kind of plugin, e.g. source, destination + kind Kind + // Team name of author of the plugin, e.g. cloudquery, vercel, github, etc + team string + // Version of the plugin + version string + // targets to build plugin for + targets []BuildTarget + // Called upon init call to validate and init configuration + newClient NewClientFunc + // Logger to call, this logger is passed to the serve.Serve Client, if not defined Serve will create one instead. + logger zerolog.Logger + // mu is a mutex that limits the number of concurrent init/syncs (can only be one at a time) + mu sync.Mutex + // client is the initialized session client + client Client + // spec is the spec the client was initialized with + spec any + // NoInternalColumns if set to true will not add internal columns to tables such as _cq_id and _cq_parent_id + // useful for sources such as PostgreSQL and other databases + internalColumns bool + // schema is the JSONSchema of the plugin spec + schema string + // validator object to validate specs + schemaValidator *jsonschema.Schema + // skips the usage client + skipUsageClient bool + // skips table validation + skipTableValidation bool + // the invocation ID for the current execution + invocationID string + // Method to test connection given a spec + testConnFn ConnectionTester + // whether to force a specific aws config + overrideConfig *aws.Config +} + +// NewPlugin returns a new CloudQuery Plugin with the given name, version and implementation. +// Depending on the options, it can be a write-only plugin, read-only plugin, or both. +func NewPlugin(name string, version string, newClient NewClientFunc, options ...Option) *Plugin { + p := Plugin{ + name: name, + version: version, + internalColumns: true, + newClient: newClient, + targets: DefaultBuildTargets, + testConnFn: UnimplementedTestConnectionFn, + } + for _, opt := range options { + opt(&p) + } + if p.schema != "" { + schemaValidator, err := JSONSchemaValidator(p.schema) + if err != nil { + panic(fmt.Errorf("failed to compile plugin JSONSchema: %w", err)) + } + p.schemaValidator = schemaValidator + } + + return &p +} + +// InvocationID returns the invocation ID for the current execution +func (p *Plugin) InvocationID() string { + return p.invocationID +} + +// Name returns the name of this plugin +func (p *Plugin) Name() string { + return p.name +} + +// Kind returns the kind of this plugin +func (p *Plugin) Kind() Kind { + return p.kind +} + +// Team returns the name of the team that authored this plugin +func (p *Plugin) Team() string { + return p.team +} + +// Version returns the version of this plugin +func (p *Plugin) Version() string { + return p.version +} + +func (p *Plugin) JSONSchema() string { + return p.schema +} + +func (p *Plugin) Meta() Meta { + return Meta{ + Team: p.team, + Kind: cqapi.PluginKind(p.kind), + Name: p.name, + SkipUsageClient: p.skipUsageClient, + } +} + +// SetSkipUsageClient sets whether the usage client should be skipped +func (p *Plugin) SetSkipUsageClient(v bool) { + p.skipUsageClient = v +} + +// SetSkipTableValidation sets whether table validation should be skipped +func (p *Plugin) SetSkipTableValidation(v bool) { + p.skipTableValidation = v +} + +type OnBeforeSender interface { + OnBeforeSend(context.Context, message.SyncMessage) (message.SyncMessage, error) +} + +// OnBeforeSend gets called before every message is sent to the destination. A plugin client +// that implements the OnBeforeSender interface will have this method called. +func (p *Plugin) OnBeforeSend(ctx context.Context, msg message.SyncMessage) (message.SyncMessage, error) { + // This method is called once for every message, so it is on the hot path, and we should be careful about its performance. + // However, most recent versions of Go have optimized type assertions and type switches to be very fast, so + // we use them here without expecting a significant impact on performance. + // See: https://stackoverflow.com/questions/28024884/does-a-type-assertion-type-switch-have-bad-performance-is-slow-in-go + if v, ok := p.client.(OnBeforeSender); ok { + return v.OnBeforeSend(ctx, msg) + } + return msg, nil +} + +// OnSyncFinisher is an interface that can be implemented by a plugin client to be notified when a sync finishes. +type OnSyncFinisher interface { + OnSyncFinish(context.Context) error +} + +// OnSyncFinish gets called after a sync finishes. +func (p *Plugin) OnSyncFinish(ctx context.Context) error { + if v, ok := p.client.(OnSyncFinisher); ok { + return v.OnSyncFinish(ctx) + } + return nil +} + +func (p *Plugin) Targets() []BuildTarget { + return p.targets +} + +func (p *Plugin) SetLogger(logger zerolog.Logger) { + p.logger = logger.With().Str("module", p.name+"-"+string(p.Kind())).Logger() +} + +func (p *Plugin) Tables(ctx context.Context, options TableOptions) (schema.Tables, error) { + if p.client == nil { + return nil, fmt.Errorf("plugin not initialized") + } + tables, err := p.client.Tables(ctx, options) + if err != nil { + return nil, fmt.Errorf("failed to get tables: %w", err) + } + return tables, nil +} + +// Init initializes the plugin with the given spec. +func (p *Plugin) Init(ctx context.Context, spec []byte, options NewClientOptions) error { + if !p.mu.TryLock() { + return fmt.Errorf("plugin already in use") + } + defer p.mu.Unlock() + var err error + + options.PluginMeta = p.Meta() + + clientLogger := p.logger + + p.invocationID = options.InvocationID + if options.InvocationID != "" { + clientLogger = p.logger.With().Str("invocation_id", p.invocationID).Logger() + } + + p.client, err = p.newClient(ctx, clientLogger, spec, options, p.overrideConfig) + if err != nil { + return fmt.Errorf("failed to initialize client: %w", err) + } + + if err := p.validate(ctx); err != nil { + return fmt.Errorf("failed to validate tables: %w", err) + } + + p.spec = spec + + return nil +} + +func (p *Plugin) Close(ctx context.Context) error { + if !p.mu.TryLock() { + return fmt.Errorf("plugin already in use") + } + defer p.mu.Unlock() + if p.client == nil { + return nil + } + return p.client.Close(ctx) +} diff --git a/plugin-sdk/plugin/plugin_destination.go b/plugin-sdk/plugin/plugin_destination.go new file mode 100644 index 00000000..59fbbf79 --- /dev/null +++ b/plugin-sdk/plugin/plugin_destination.go @@ -0,0 +1,55 @@ +package plugin + +import ( + "context" + "fmt" + + "github.com/apache/arrow/go/v16/arrow" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +type DestinationClient interface { + Close(ctx context.Context) error + Read(ctx context.Context, table *schema.Table, res chan<- arrow.Record) error + Write(ctx context.Context, res <-chan message.WriteMessage) error +} + +// writeOne is currently used mostly for testing, so it's not a public api +func (p *Plugin) writeOne(ctx context.Context, resource message.WriteMessage) error { + resources := []message.WriteMessage{resource} + return p.WriteAll(ctx, resources) +} + +// WriteAll is currently used mostly for testing, so it's not a public api +func (p *Plugin) WriteAll(ctx context.Context, resources []message.WriteMessage) error { + ch := make(chan message.WriteMessage, len(resources)) + for _, resource := range resources { + ch <- resource + } + close(ch) + return p.Write(ctx, ch) +} + +func (p *Plugin) Write(ctx context.Context, res <-chan message.WriteMessage) error { + if p.client == nil { + return fmt.Errorf("plugin is not initialized. call Init first") + } + return p.client.Write(ctx, res) +} + +// Read is read data from the requested table to the given channel, returned in the same format as the table +func (p *Plugin) Read(ctx context.Context, table *schema.Table, res chan<- arrow.Record) error { + if !p.mu.TryLock() { + return fmt.Errorf("plugin already in use") + } + defer p.mu.Unlock() + if p.client == nil { + return fmt.Errorf("plugin not initialized. call Init() first") + } + if err := p.client.Read(ctx, table, res); err != nil { + return fmt.Errorf("failed to read: %w", err) + } + return nil +} diff --git a/plugin-sdk/plugin/plugin_package.go b/plugin-sdk/plugin/plugin_package.go new file mode 100644 index 00000000..88ba2f8f --- /dev/null +++ b/plugin-sdk/plugin/plugin_package.go @@ -0,0 +1,84 @@ +package plugin + +import ( + "errors" +) + +const ( + GoOSLinux = "linux" + GoOSWindows = "windows" + GoOSDarwin = "darwin" + + GoArchAmd64 = "amd64" + GoArchArm64 = "arm64" +) + +type Kind string + +const ( + KindSource Kind = "source" + KindDestination Kind = "destination" +) + +func (k Kind) Validate() error { + switch k { + case KindSource, KindDestination: + return nil + default: + return errors.New("invalid plugin kind: must be one of source, destination") + } +} + +type PackageType string + +const ( + PackageTypeNative PackageType = "native" +) + +type BuildTarget struct { + OS string `json:"os"` + Arch string `json:"arch"` + CGO bool `json:"cgo"` + Env []string `json:"env"` +} + +func (t BuildTarget) EnvVariables() []string { + variables := append(t.cgoEnvVariables(), "GOOS="+t.OS, "GOARCH="+t.Arch) + return append(variables, t.Env...) +} + +func (t BuildTarget) cgoEnvVariables() []string { + // default is to tool at the param. Can be overridden by adding `CGO_ENABLED=1` to BuildTarget.Env + if !t.CGO { + return []string{"CGO_ENABLED=0"} + } + + switch t.OS { + case GoOSWindows: + return []string{"CGO_ENABLED=1", "CC=x86_64-w64-mingw32-gcc", "CXX=x86_64-w64-mingw32-g++"} + case GoOSDarwin: + return []string{"CGO_ENABLED=1", "CC=o64-clang", "CXX=o64-clang++"} + case GoOSLinux: + // nop, see below + default: + return []string{"CGO_ENABLED=1"} + } + + // linux + switch t.Arch { + case GoArchAmd64: + return []string{"CGO_ENABLED=1", "CC=gcc", "CXX=g++"} + case GoArchArm64: + return []string{"CGO_ENABLED=1", "CC=aarch64-linux-gnu-gcc", "CXX=aarch64-linux-gnu-g++"} + default: + return []string{"CGO_ENABLED=1"} + } +} + +var DefaultBuildTargets = []BuildTarget{ + {OS: GoOSLinux, Arch: GoArchAmd64}, + {OS: GoOSLinux, Arch: GoArchArm64}, + {OS: GoOSWindows, Arch: GoArchAmd64}, + {OS: GoOSDarwin, Arch: GoArchAmd64}, + {OS: GoOSDarwin, Arch: GoArchArm64}, +} diff --git a/plugin-sdk/plugin/plugin_read.go b/plugin-sdk/plugin/plugin_read.go new file mode 100644 index 00000000..2dcf1a7f --- /dev/null +++ b/plugin-sdk/plugin/plugin_read.go @@ -0,0 +1,34 @@ +package plugin + +import ( + "context" + + "github.com/apache/arrow/go/v16/arrow" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +// readAll is used in tests to read all records from a table. +func (p *Plugin) readAll(ctx context.Context, table *schema.Table) ([]arrow.Record, error) { + var err error + ch := make(chan arrow.Record) + go func() { + defer close(ch) + err = p.client.Read(ctx, table, ch) + }() + // nolint:prealloc + var records []arrow.Record + for record := range ch { + records = append(records, sliceToSingleRowRecord(record)...) + } + + return records, err +} + +func sliceToSingleRowRecord(record arrow.Record) []arrow.Record { + result := make([]arrow.Record, record.NumRows()) + for i := int64(0); i < record.NumRows(); i++ { + result[i] = record.NewSlice(i, i+1) + } + return result +} diff --git a/plugin-sdk/plugin/plugin_source.go b/plugin-sdk/plugin/plugin_source.go new file mode 100644 index 00000000..ca6293ae --- /dev/null +++ b/plugin-sdk/plugin/plugin_source.go @@ -0,0 +1,100 @@ +package plugin + +import ( + "context" + "fmt" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/rs/zerolog" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/glob" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +type BackendOptions struct { + TableName string + Connection string +} + +type SyncOptions struct { + Tables []string + SkipTables []string + SkipDependentTables bool + DeterministicCQID bool + BackendOptions *BackendOptions +} + +type SourceClient interface { + Close(ctx context.Context) error + Tables(ctx context.Context, options TableOptions) (schema.Tables, error) + Sync(ctx context.Context, options SyncOptions, res chan<- message.SyncMessage, resourceCh chan<- *schema.Resource) error +} + +func MatchesTable(name string, includeTablesPattern []string, skipTablesPattern []string) bool { + for _, pattern := range skipTablesPattern { + if glob.Glob(pattern, name) { + return false + } + } + for _, pattern := range includeTablesPattern { + if glob.Glob(pattern, name) { + return true + } + } + return false +} + +type NewSourceClientFunc func(context.Context, zerolog.Logger, any) (SourceClient, error) + +// NewSourcePlugin returns a new CloudQuery Plugin with the given name, version and implementation. +// Source plugins only support read operations. For Read & Write plugin use NewPlugin. +func NewSourcePlugin(name string, version string, newClient NewSourceClientFunc, options ...Option) *Plugin { + newClientWrapper := func(ctx context.Context, logger zerolog.Logger, spec []byte, _ NewClientOptions, _ *aws.Config) (Client, error) { + sourceClient, err := newClient(ctx, logger, spec) + if err != nil { + return nil, err + } + wrapperClient := struct { + SourceClient + UnimplementedDestination + }{ + SourceClient: sourceClient, + } + return wrapperClient, nil + } + return NewPlugin(name, version, newClientWrapper, options...) +} + +func (p *Plugin) SyncAll(ctx context.Context, options SyncOptions) (message.SyncMessages, error) { + var err error + ch := make(chan message.SyncMessage) + resourceCh := make(chan *schema.Resource) + go func() { + defer close(ch) + err = p.Sync(ctx, options, ch, resourceCh) + }() + // nolint:prealloc + var resources message.SyncMessages + for resource := range ch { + resources = append(resources, resource) + } + return resources, err +} + +// Sync is syncing data from the requested tables in spec to the given channel +func (p *Plugin) Sync(ctx context.Context, options SyncOptions, res chan<- message.SyncMessage, resourceCh chan<- *schema.Resource) error { + if !p.mu.TryLock() { + return fmt.Errorf("plugin already in use") + } + defer p.mu.Unlock() + if p.client == nil { + return fmt.Errorf("plugin not initialized. call Init() first") + } + + if err := p.client.Sync(ctx, options, res, resourceCh); err != nil { + return fmt.Errorf("failed to sync unmanaged client: %w", err) + } + + return nil +} diff --git a/plugin-sdk/plugin/plugin_test.go b/plugin-sdk/plugin/plugin_test.go new file mode 100644 index 00000000..f8fe2d7c --- /dev/null +++ b/plugin-sdk/plugin/plugin_test.go @@ -0,0 +1,105 @@ +package plugin + +import ( + "context" + "testing" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/rs/zerolog" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +type testPluginClient struct { + messages message.SyncMessages + resources schema.Resources +} + +func newTestPluginClient(context.Context, zerolog.Logger, []byte, NewClientOptions, *aws.Config) (Client, error) { + return &testPluginClient{}, nil +} + +func (*testPluginClient) GetSpec() any { + return &struct{}{} +} + +func (*testPluginClient) Tables(context.Context, TableOptions) (schema.Tables, error) { + return schema.Tables{}, nil +} + +func (*testPluginClient) Read(context.Context, *schema.Table, chan<- arrow.Record) error { + return nil +} + +func (c *testPluginClient) Sync(_ context.Context, _ SyncOptions, res chan<- message.SyncMessage, resourceCh chan<- *schema.Resource) error { + for _, msg := range c.messages { + res <- msg + } + + for _, r := range c.resources { + resourceCh <- r + } + return nil +} + +func (c *testPluginClient) Write(_ context.Context, res <-chan message.WriteMessage) error { + for msg := range res { + switch m := msg.(type) { + case *message.WriteMigrateTable: + c.messages = append(c.messages, &message.SyncMigrateTable{ + Table: m.Table, + }) + case *message.WriteInsert: + c.messages = append(c.messages, &message.SyncInsert{ + Record: m.Record, + }) + default: + panic("unknown message") + } + } + return nil +} + +func (*testPluginClient) Close(context.Context) error { + return nil +} + +func TestPluginSuccess(t *testing.T) { + ctx := context.Background() + p := NewPlugin("test", "v1.0.0", newTestPluginClient) + if err := p.Init(ctx, []byte(""), NewClientOptions{}); err != nil { + t.Fatal(err) + } + tables, err := p.Tables(ctx, TableOptions{}) + if err != nil { + t.Fatal(err) + } + if len(tables) != 0 { + t.Fatal("expected 0 tables") + } + if err := p.WriteAll(ctx, nil); err != nil { + t.Fatal(err) + } + if err := p.WriteAll(ctx, []message.WriteMessage{ + &message.WriteMigrateTable{}, + }); err != nil { + t.Fatal(err) + } + if len(p.client.(*testPluginClient).messages) != 1 { + t.Fatal("expected 1 message") + } + + messages, err := p.SyncAll(ctx, SyncOptions{}) + if err != nil { + t.Fatal(err) + } + if len(messages) != 1 { + t.Fatal("expected 1 message") + } + + if err := p.Close(ctx); err != nil { + t.Fatal(err) + } +} diff --git a/plugin-sdk/plugin/plugin_testconn.go b/plugin-sdk/plugin/plugin_testconn.go new file mode 100644 index 00000000..e02128f7 --- /dev/null +++ b/plugin-sdk/plugin/plugin_testconn.go @@ -0,0 +1,46 @@ +package plugin + +import ( + "context" + + "github.com/rs/zerolog" +) + +type TestConnError struct { + Code string + Message error +} + +func NewTestConnError(code string, err error) *TestConnError { + return &TestConnError{ + Code: code, + Message: err, + } +} + +func (e *TestConnError) Error() string { + return e.Message.Error() +} + +func (e *TestConnError) Unwrap() error { + return e.Message +} + +func (e *TestConnError) Is(err error) bool { + if err2, ok := err.(*TestConnError); ok { + return e.Code == err2.Code + } + return false +} + +type ConnectionTester func(ctx context.Context, logger zerolog.Logger, spec []byte) error + +func (p *Plugin) TestConnection(ctx context.Context, logger zerolog.Logger, spec []byte) error { + return p.testConnFn(ctx, logger, spec) +} + +func UnimplementedTestConnectionFn(context.Context, zerolog.Logger, []byte) error { + return ErrNotImplemented +} + +var _ ConnectionTester = UnimplementedTestConnectionFn diff --git a/plugin-sdk/plugin/sort.go b/plugin-sdk/plugin/sort.go new file mode 100644 index 00000000..03946d7e --- /dev/null +++ b/plugin-sdk/plugin/sort.go @@ -0,0 +1,36 @@ +package plugin + +import ( + "sort" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +// sort records by specific column. This is intended for testing purposes only. +// Because "id" is auto-incrementing in the test data generator, if passed "id" +// this should result in records being returned in insertion order. +// nolint:unparam +func sortRecords(table *schema.Table, records []arrow.Record, columnName string) { + sch := table.ToArrowSchema() + if !sch.HasField(columnName) { + panic("table has no '" + columnName + "' column to sort on") + } + colIndex := sch.FieldIndices(columnName)[0] + sort.Slice(records, func(i, j int) bool { + switch records[i].Column(colIndex).DataType().(type) { + case *arrow.Int64Type: + v1 := records[i].Column(colIndex).(*array.Int64).Value(0) + v2 := records[j].Column(colIndex).(*array.Int64).Value(0) + return v1 < v2 + case *arrow.StringType: + v1 := records[i].Column(colIndex).(*array.String).Value(0) + v2 := records[j].Column(colIndex).(*array.String).Value(0) + return v1 < v2 + default: + panic("unsupported type for sorting") + } + }) +} diff --git a/plugin-sdk/plugin/testing.go b/plugin-sdk/plugin/testing.go new file mode 100644 index 00000000..b4b8cb46 --- /dev/null +++ b/plugin-sdk/plugin/testing.go @@ -0,0 +1,12 @@ +package plugin + +import ( + "fmt" + "time" +) + +// tableNameForTest returns a table name that is unique to the test run. It adds the current unix second +// and a random number between 0 and 1000 as a suffix. +func (s *WriterTestSuite) tableNameForTest(name string) string { + return fmt.Sprintf("cq_%s_test_%d_%04d", name, time.Now().Unix(), s.rand.Intn(1000)) +} diff --git a/plugin-sdk/plugin/testing_validation.go b/plugin-sdk/plugin/testing_validation.go new file mode 100644 index 00000000..4b09d631 --- /dev/null +++ b/plugin-sdk/plugin/testing_validation.go @@ -0,0 +1,18 @@ +package plugin + +import ( + "testing" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +func ValidateNoEmptyColumns(t *testing.T, tables schema.Tables, messages message.SyncMessages) { + for _, table := range tables.FlattenTables() { + records := messages.GetInserts().GetRecordsForTable(table) + emptyColumns := schema.FindEmptyColumns(table, records) + if len(emptyColumns) > 0 { + t.Fatalf("found empty column(s): %v in %s", emptyColumns, table.Name) + } + } +} diff --git a/plugin-sdk/plugin/testing_write.go b/plugin-sdk/plugin/testing_write.go new file mode 100644 index 00000000..0bfd8e43 --- /dev/null +++ b/plugin-sdk/plugin/testing_write.go @@ -0,0 +1,207 @@ +package plugin + +import ( + "context" + "math/rand" + "testing" + + "github.com/apache/arrow/go/v16/arrow" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +type WriterTestSuite struct { + tests WriterTestSuiteTests + + plugin *Plugin + + // AllowNull is a custom func to determine whether a data type may be correctly represented as null. + // Destinations that have problems representing some data types should provide a custom implementation here. + // If this param is empty, the default is to allow all data types to be nullable. + // When the value returned by this func is `true` the comparison is made with the empty value instead of null. + allowNull AllowNullFunc + + // IgnoreNullsInLists allows stripping null values from lists before comparison. + // Destination setups that don't support nulls in lists should set this to true. + ignoreNullsInLists bool + + // useHomogeneousTypes use the same type for all items within an array - useful for destinations that don't support mixed types. + useHomogeneousTypes bool + + // genDataOptions define how to generate test data and which data types to skip + genDatOptions schema.TestSourceOptions + + // random seed to use + randSeed int64 + + // rand.Rand + rand *rand.Rand +} + +// SafeMigrations defines which migrations are supported by the plugin in safe migrate mode +type SafeMigrations struct { + AddColumn bool + AddColumnNotNull bool + RemoveColumn bool + RemoveColumnNotNull bool + ChangeColumn bool + RemoveUniqueConstraint bool + MovePKToCQOnly bool +} + +// Migrations defines which migrations should be skipped completely +type Migrations struct { + RemoveUniqueConstraint bool + MovePKToCQOnly bool +} + +// WriteTests defines which tests should be skipped in the write test suite +type WriteTests struct { + DuplicatePK bool +} + +type WriterTestSuiteTests struct { + // SkipUpsert skips testing with message.Insert and Upsert=true. + // Usually when a destination is not supporting primary keys + SkipUpsert bool + + // SkipDeleteStale skips testing message.Delete events. + SkipDeleteStale bool + + // SkipDeleteRecord skips testing message.DeleteRecord events. + SkipDeleteRecord bool + + // SkipAppend skips testing message.Insert and Upsert=false. + SkipInsert bool + + // SkipMigrate skips testing migration + SkipMigrate bool + + // SafeMigrations defines which tests should work with force migration + // and which should pass with safe migration + SafeMigrations SafeMigrations + + SkipSpecificMigrations Migrations + + SkipSpecificWriteTests WriteTests +} + +type NewPluginFunc func() *Plugin + +func WithTestSourceAllowNull(allowNull func(arrow.DataType) bool) func(o *WriterTestSuite) { + return func(o *WriterTestSuite) { + o.allowNull = allowNull + } +} + +func WithTestIgnoreNullsInLists() func(o *WriterTestSuite) { + return func(o *WriterTestSuite) { + o.ignoreNullsInLists = true + } +} + +func WithTestDataOptions(opts schema.TestSourceOptions) func(o *WriterTestSuite) { + return func(o *WriterTestSuite) { + o.genDatOptions = opts + } +} + +func WithRandomSeed(seed int64) func(o *WriterTestSuite) { + return func(o *WriterTestSuite) { + o.randSeed = seed + } +} + +func WithHomogeneousTypes() func(o *WriterTestSuite) { + return func(o *WriterTestSuite) { + o.useHomogeneousTypes = true + } +} + +func TestWriterSuiteRunner(t *testing.T, p *Plugin, tests WriterTestSuiteTests, opts ...func(o *WriterTestSuite)) { + suite := &WriterTestSuite{ + tests: tests, + plugin: p, + } + + for _, opt := range opts { + opt(suite) + } + + suite.rand = rand.New(rand.NewSource(suite.randSeed)) + + ctx := context.Background() + + t.Run("TestUpsert", func(t *testing.T) { + if suite.tests.SkipUpsert { + t.Skip("skipping " + t.Name()) + } + t.Run("Basic", func(t *testing.T) { + if err := suite.testUpsertBasic(ctx); err != nil { + t.Fatal(err) + } + }) + t.Run("All", func(t *testing.T) { + if err := suite.testUpsertAll(ctx); err != nil { + t.Fatal(err) + } + }) + + t.Run("Duplicate PKs", func(t *testing.T) { + if tests.SkipSpecificWriteTests.DuplicatePK { + t.Skip("skipping " + t.Name()) + } + if err := suite.testInsertDuplicatePK(ctx); err != nil { + t.Fatal(err) + } + }) + }) + + t.Run("TestInsert", func(t *testing.T) { + if suite.tests.SkipInsert { + t.Skip("skipping " + t.Name()) + } + t.Run("Basic", func(t *testing.T) { + if err := suite.testInsertBasic(ctx); err != nil { + t.Fatal(err) + } + }) + t.Run("All", func(t *testing.T) { + if err := suite.testInsertAll(ctx); err != nil { + t.Fatal(err) + } + }) + }) + + t.Run("TestDeleteStale", func(t *testing.T) { + if suite.tests.SkipDeleteStale { + t.Skip("skipping " + t.Name()) + } + t.Run("Basic", func(t *testing.T) { + suite.testDeleteStaleBasic(ctx, t) + }) + t.Run("All", func(t *testing.T) { + suite.testDeleteStaleAll(ctx, t) + }) + }) + + t.Run("TestDeleteRecord", func(t *testing.T) { + if suite.tests.SkipDeleteRecord { + t.Skip("skipping " + t.Name()) + } + t.Run("Basic", func(t *testing.T) { + suite.testDeleteRecordBasic(ctx, t) + }) + t.Run("DeleteAll", func(t *testing.T) { + suite.testDeleteAllRecords(ctx, t) + }) + }) + + t.Run("TestMigrate", func(t *testing.T) { + if suite.tests.SkipMigrate { + t.Skip("skipping " + t.Name()) + } + suite.testMigrate(ctx, t, false) + suite.testMigrate(ctx, t, true) + }) +} diff --git a/plugin-sdk/plugin/testing_write_delete.go b/plugin-sdk/plugin/testing_write_delete.go new file mode 100644 index 00000000..d367aacf --- /dev/null +++ b/plugin-sdk/plugin/testing_write_delete.go @@ -0,0 +1,302 @@ +package plugin + +import ( + "context" + "testing" + "time" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +func (s *WriterTestSuite) testDeleteStaleBasic(ctx context.Context, t *testing.T) { + r := require.New(t) + tableName := s.tableNameForTest("delete_basic") + syncTime := time.Now().UTC().Truncate(s.genDatOptions.TimePrecision). + Truncate(time.Microsecond) // https://github.com/golang/go/issues/41087 + table := &schema.Table{ + Name: tableName, + Columns: schema.ColumnList{ + schema.Column{Name: "id", Type: arrow.PrimitiveTypes.Int64, PrimaryKey: true, NotNull: true}, + schema.CqSourceNameColumn, + schema.CqSyncTimeColumn, + }, + } + r.NoErrorf(s.plugin.writeOne(ctx, &message.WriteMigrateTable{Table: table}), "failed to create table") + const sourceName = "source-test" + + bldr := array.NewRecordBuilder(memory.DefaultAllocator, table.ToArrowSchema()) + bldr.Field(0).(*array.Int64Builder).Append(0) + bldr.Field(1).(*array.StringBuilder).Append(sourceName) + bldr.Field(2).(*array.TimestampBuilder).AppendTime(syncTime) + record1 := bldr.NewRecord() + + r.NoErrorf(s.plugin.writeOne(ctx, &message.WriteInsert{Record: record1}), "failed to insert record") + record1 = s.handleNulls(record1) // we process nulls after writing + + records, err := s.plugin.readAll(ctx, table) + r.NoErrorf(err, "failed to read") + r.EqualValuesf(1, TotalRows(records), "unexpected amount of items") + + r.NoErrorf(s.plugin.writeOne(ctx, &message.WriteDeleteStale{ + TableName: table.Name, + SourceName: sourceName, + SyncTime: syncTime, + }), "failed to delete stale records") + + records, err = s.plugin.readAll(ctx, table) + r.NoErrorf(err, "failed to read after delete stale") + r.EqualValuesf(1, TotalRows(records), "unexpected amount of items after delete stale") + r.Emptyf(RecordsDiff(table.ToArrowSchema(), records, []arrow.Record{record1}), "record differs after delete stale") + + bldr.Field(0).(*array.Int64Builder).Append(1) + bldr.Field(1).(*array.StringBuilder).Append(sourceName) + bldr.Field(2).(*array.TimestampBuilder).AppendTime(syncTime.Add(time.Second)) + record2 := bldr.NewRecord() + + r.NoErrorf(s.plugin.writeOne(ctx, &message.WriteInsert{Record: record2}), "failed to insert second record") + record2 = s.handleNulls(record2) // we process nulls after writing + + records, err = s.plugin.readAll(ctx, table) + r.NoErrorf(err, "failed to read second time") + sortRecords(table, records, "id") + r.EqualValuesf(2, TotalRows(records), "unexpected amount of items second time") + r.Emptyf(RecordsDiff(table.ToArrowSchema(), records, []arrow.Record{record1, record2}), "record differs after delete stale") + + r.NoErrorf(s.plugin.writeOne(ctx, &message.WriteDeleteStale{ + TableName: table.Name, + SourceName: sourceName, + SyncTime: syncTime.Add(time.Second), + }), "failed to delete stale records second time") + + records, err = s.plugin.readAll(ctx, table) + r.NoErrorf(err, "failed to read after second delete stale") + r.EqualValuesf(1, TotalRows(records), "unexpected amount of items after second delete stale") + r.Emptyf(RecordsDiff(table.ToArrowSchema(), records, []arrow.Record{record2}), "record differs after second delete stale") +} + +func (s *WriterTestSuite) testDeleteStaleAll(ctx context.Context, t *testing.T) { + const rowsPerRecord = 10 + + r := require.New(t) + tableName := s.tableNameForTest("delete_all") + // https://github.com/golang/go/issues/41087 + syncTime := time.Now().UTC().Truncate(time.Microsecond) + table := schema.TestTable(tableName, s.genDatOptions) + table.Columns = append(schema.ColumnList{schema.CqSourceNameColumn, schema.CqSyncTimeColumn}, table.Columns...) + table.Columns[table.Columns.Index("id")].PrimaryKey = true + r.NoErrorf(s.plugin.writeOne(ctx, &message.WriteMigrateTable{Table: table}), "failed to create table") + + tg := schema.NewTestDataGenerator(0) + normalRecord := tg.Generate(table, schema.GenTestDataOptions{ + MaxRows: rowsPerRecord, + TimePrecision: s.genDatOptions.TimePrecision, + SourceName: "test", + SyncTime: syncTime, // Generate call may truncate the value further based on the options + UseHomogeneousType: s.useHomogeneousTypes, + }) + r.NoErrorf(s.plugin.writeOne(ctx, &message.WriteInsert{Record: normalRecord}), "failed to insert record") + normalRecord = s.handleNulls(normalRecord) // we process nulls after writing + + readRecords, err := s.plugin.readAll(ctx, table) + r.NoErrorf(err, "failed to read") + r.EqualValuesf(rowsPerRecord, TotalRows(readRecords), "unexpected amount of items after read") + + r.NoErrorf(s.plugin.writeOne(ctx, &message.WriteDeleteStale{ + TableName: table.Name, + SourceName: "test", + SyncTime: syncTime, // Generate call may truncate the value further based on the options + }), "failed to delete stale records") + + readRecords, err = s.plugin.readAll(ctx, table) + r.NoErrorf(err, "failed to read after delete stale") + r.EqualValuesf(rowsPerRecord, TotalRows(readRecords), "unexpected amount of items after delete stale") + + // https://github.com/golang/go/issues/41087 + syncTime = time.Now().UTC().Truncate(time.Microsecond) + nullRecord := tg.Generate(table, schema.GenTestDataOptions{ + MaxRows: rowsPerRecord, + TimePrecision: s.genDatOptions.TimePrecision, + NullRows: true, + SourceName: "test", + SyncTime: syncTime, // Generate call may truncate the value further based on the options + UseHomogeneousType: s.useHomogeneousTypes, + }) + r.NoErrorf(s.plugin.writeOne(ctx, &message.WriteInsert{Record: nullRecord}), "failed to insert record second time") + nullRecord = s.handleNulls(nullRecord) // we process nulls after writing + + readRecords, err = s.plugin.readAll(ctx, table) + r.NoErrorf(err, "failed to read second time") + sortRecords(table, readRecords, "id") + r.EqualValuesf(2*rowsPerRecord, TotalRows(readRecords), "unexpected amount of items after second read") + r.Emptyf(RecordsDiff(table.ToArrowSchema(), readRecords, []arrow.Record{normalRecord, nullRecord}), "record differs") + + r.NoErrorf(s.plugin.writeOne(ctx, &message.WriteDeleteStale{ + TableName: table.Name, + SourceName: "test", + SyncTime: syncTime, // Generate call may truncate the value further based on the options + }), "failed to delete stale records second time") + + readRecords, err = s.plugin.readAll(ctx, table) + r.NoErrorf(err, "failed to read after second delete stale") + sortRecords(table, readRecords, "id") + r.EqualValuesf(rowsPerRecord, TotalRows(readRecords), "unexpected amount of items after second delete stale") + r.Emptyf(RecordsDiff(table.ToArrowSchema(), readRecords, []arrow.Record{nullRecord}), "record differs") +} + +func (s *WriterTestSuite) testDeleteRecordBasic(ctx context.Context, t *testing.T) { + r := require.New(t) + tableName := s.tableNameForTest("delete_all_rows") + syncTime := time.Now().UTC().Truncate(s.genDatOptions.TimePrecision). + Truncate(time.Microsecond) // https://github.com/golang/go/issues/41087 + table := &schema.Table{ + Name: tableName, + Columns: schema.ColumnList{ + schema.Column{Name: "id", Type: arrow.PrimitiveTypes.Int64, PrimaryKey: true, NotNull: true}, + schema.CqSourceNameColumn, + schema.CqSyncTimeColumn, + }, + } + r.NoErrorf(s.plugin.writeOne(ctx, &message.WriteMigrateTable{Table: table}), "failed to create table") + const sourceName = "source-test" + + bldr := array.NewRecordBuilder(memory.DefaultAllocator, table.ToArrowSchema()) + bldr.Field(0).(*array.Int64Builder).Append(0) + bldr.Field(1).(*array.StringBuilder).Append(sourceName) + bldr.Field(2).(*array.TimestampBuilder).AppendTime(syncTime) + record1 := bldr.NewRecord() + + r.NoErrorf(s.plugin.writeOne(ctx, &message.WriteInsert{Record: record1}), "failed to insert record") + record1 = s.handleNulls(record1) // we process nulls after writing + + records, err := s.plugin.readAll(ctx, table) + r.NoErrorf(err, "failed to read") + r.EqualValuesf(1, TotalRows(records), "unexpected amount of items") + + // create value for delete statement but nothing will be deleted because ID value isn't present + bldrDeleteNoMatch := array.NewRecordBuilder(memory.DefaultAllocator, (&schema.Table{ + Name: tableName, + Columns: schema.ColumnList{ + schema.Column{Name: "id", Type: arrow.PrimitiveTypes.Int64}, + }, + }).ToArrowSchema()) + bldrDeleteNoMatch.Field(0).(*array.Int64Builder).Append(1) + deleteValue := bldrDeleteNoMatch.NewRecord() + + r.NoErrorf(s.plugin.writeOne(ctx, &message.WriteDeleteRecord{ + DeleteRecord: message.DeleteRecord{ + TableName: table.Name, + WhereClause: message.PredicateGroups{ + { + GroupingType: "AND", + Predicates: []message.Predicate{ + { + Operator: "eq", + Column: "id", + Record: deleteValue, + }, + }, + }, + }, + }, + }), "failed to delete record no match") + + records, err = s.plugin.readAll(ctx, table) + r.NoErrorf(err, "failed to read after delete with no match") + r.EqualValuesf(1, TotalRows(records), "unexpected amount of items after delete with no match") + r.Emptyf(RecordsDiff(table.ToArrowSchema(), records, []arrow.Record{record1}), "record differs after delete with no match") + + // create value for delete statement will be delete One record + bldrDeleteMatch := array.NewRecordBuilder(memory.DefaultAllocator, (&schema.Table{ + Name: tableName, + Columns: schema.ColumnList{ + schema.Column{Name: "id", Type: arrow.PrimitiveTypes.Int64}, + }, + }).ToArrowSchema()) + bldrDeleteMatch.Field(0).(*array.Int64Builder).Append(0) + deleteValue = bldrDeleteMatch.NewRecord() + + r.NoErrorf(s.plugin.writeOne(ctx, &message.WriteDeleteRecord{ + DeleteRecord: message.DeleteRecord{ + TableName: table.Name, + WhereClause: message.PredicateGroups{ + { + GroupingType: "AND", + Predicates: []message.Predicate{ + { + Operator: "eq", + Column: "id", + Record: deleteValue, + }, + }, + }, + }, + }, + }), "failed to delete record no match") + + records, err = s.plugin.readAll(ctx, table) + r.NoErrorf(err, "failed to read after delete with match") + r.EqualValuesf(0, TotalRows(records), "unexpected amount of items after delete with match") +} + +func (s *WriterTestSuite) testDeleteAllRecords(ctx context.Context, t *testing.T) { + r := require.New(t) + tableName := s.tableNameForTest("delete_all_records") + syncTime := time.Now().UTC().Truncate(s.genDatOptions.TimePrecision). + Truncate(time.Microsecond) // https://github.com/golang/go/issues/41087 + table := &schema.Table{ + Name: tableName, + Columns: schema.ColumnList{ + schema.Column{Name: "id", Type: arrow.PrimitiveTypes.Int64, PrimaryKey: true, NotNull: true}, + schema.CqSourceNameColumn, + schema.CqSyncTimeColumn, + }, + } + r.NoErrorf(s.plugin.writeOne(ctx, &message.WriteMigrateTable{Table: table}), "failed to create table") + const sourceName = "source-test" + + bldr := array.NewRecordBuilder(memory.DefaultAllocator, table.ToArrowSchema()) + bldr.Field(0).(*array.Int64Builder).Append(0) + bldr.Field(1).(*array.StringBuilder).Append(sourceName) + bldr.Field(2).(*array.TimestampBuilder).AppendTime(syncTime) + record1 := bldr.NewRecord() + + r.NoErrorf(s.plugin.writeOne(ctx, &message.WriteInsert{Record: record1}), "failed to insert record") + + records, err := s.plugin.readAll(ctx, table) + r.NoErrorf(err, "failed to read") + r.EqualValuesf(1, TotalRows(records), "unexpected amount of items") + + r.NoErrorf(s.plugin.writeOne(ctx, &message.WriteDeleteRecord{ + DeleteRecord: message.DeleteRecord{ + TableName: table.Name, + }, + }), "failed to delete records") + + records, err = s.plugin.readAll(ctx, table) + r.NoErrorf(err, "failed to read after delete all records") + r.EqualValuesf(0, TotalRows(records), "unexpected amount of items after delete stale") + + bldr.Field(0).(*array.Int64Builder).Append(1) + bldr.Field(1).(*array.StringBuilder).Append(sourceName) + bldr.Field(2).(*array.TimestampBuilder).AppendTime(syncTime.Add(time.Second)) + record2 := bldr.NewRecord() + + r.NoErrorf(s.plugin.writeOne(ctx, &message.WriteInsert{Record: record2}), "failed to insert second record") + + r.NoErrorf(s.plugin.writeOne(ctx, &message.WriteDeleteRecord{ + DeleteRecord: message.DeleteRecord{ + TableName: table.Name, + }, + }), "failed to delete records second time") + + records, err = s.plugin.readAll(ctx, table) + r.NoErrorf(err, "failed to read second time") + sortRecords(table, records, "id") + r.EqualValuesf(0, TotalRows(records), "unexpected amount of items second time") +} diff --git a/plugin-sdk/plugin/testing_write_insert.go b/plugin-sdk/plugin/testing_write_insert.go new file mode 100644 index 00000000..68a7c832 --- /dev/null +++ b/plugin-sdk/plugin/testing_write_insert.go @@ -0,0 +1,142 @@ +package plugin + +import ( + "context" + "fmt" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +func TotalRows(records []arrow.Record) int64 { + totalRows := int64(0) + for _, record := range records { + totalRows += record.NumRows() + } + return totalRows +} + +func (s *WriterTestSuite) testInsertBasic(ctx context.Context) error { + tableName := s.tableNameForTest("insert_basic") + table := &schema.Table{ + Name: tableName, + Columns: []schema.Column{ + {Name: "id", Type: arrow.PrimitiveTypes.Int64, NotNull: true}, + {Name: "name", Type: arrow.BinaryTypes.String}, + }, + } + if err := s.plugin.writeOne(ctx, &message.WriteMigrateTable{ + Table: table, + }); err != nil { + return fmt.Errorf("failed to create table: %w", err) + } + + bldr := array.NewRecordBuilder(memory.DefaultAllocator, table.ToArrowSchema()) + bldr.Field(0).(*array.Int64Builder).Append(1) + bldr.Field(1).(*array.StringBuilder).Append("foo") + record := bldr.NewRecord() + + if err := s.plugin.writeOne(ctx, &message.WriteInsert{ + Record: record, + }); err != nil { + return fmt.Errorf("failed to insert record: %w", err) + } + record = s.handleNulls(record) // we process nulls after writing + + readRecords, err := s.plugin.readAll(ctx, table) + if err != nil { + return fmt.Errorf("failed to sync: %w", err) + } + + totalItems := TotalRows(readRecords) + if totalItems != 1 { + return fmt.Errorf("expected 1 item, got %d", totalItems) + } + + if err := s.plugin.writeOne(ctx, &message.WriteInsert{ + Record: record, + }); err != nil { + return fmt.Errorf("failed to insert record: %w", err) + } + + readRecords, err = s.plugin.readAll(ctx, table) + if err != nil { + return fmt.Errorf("failed to sync: %w", err) + } + sortRecords(table, readRecords, "id") + + totalItems = TotalRows(readRecords) + if totalItems != 2 { + return fmt.Errorf("expected 2 items, got %d", totalItems) + } + + if diff := RecordsDiff(table.ToArrowSchema(), readRecords, []arrow.Record{record, record}); diff != "" { + return fmt.Errorf("record[0] differs: %s", diff) + } + + return nil +} + +func (s *WriterTestSuite) testInsertAll(ctx context.Context) error { + const rowsPerRecord = 10 + tableName := s.tableNameForTest("insert_all") + table := schema.TestTable(tableName, s.genDatOptions) + if err := s.plugin.writeOne(ctx, &message.WriteMigrateTable{ + Table: table, + }); err != nil { + return fmt.Errorf("failed to create table: %w", err) + } + tg := schema.NewTestDataGenerator(0) + normalRecord := tg.Generate(table, schema.GenTestDataOptions{ + MaxRows: rowsPerRecord, + TimePrecision: s.genDatOptions.TimePrecision, + UseHomogeneousType: s.useHomogeneousTypes, + }) + if err := s.plugin.writeOne(ctx, &message.WriteInsert{ + Record: normalRecord, + }); err != nil { + return fmt.Errorf("failed to insert record: %w", err) + } + normalRecord = s.handleNulls(normalRecord) // we process nulls after writing + + readRecords, err := s.plugin.readAll(ctx, table) + if err != nil { + return fmt.Errorf("failed to sync: %w", err) + } + + totalItems := TotalRows(readRecords) + if totalItems != rowsPerRecord { + return fmt.Errorf("items expected after first insert: %d, got: %d", rowsPerRecord, totalItems) + } + + nullRecord := tg.Generate(table, schema.GenTestDataOptions{ + MaxRows: rowsPerRecord, + TimePrecision: s.genDatOptions.TimePrecision, + NullRows: true, + }) + if err := s.plugin.writeOne(ctx, &message.WriteInsert{ + Record: nullRecord, + }); err != nil { + return fmt.Errorf("failed to insert record: %w", err) + } + nullRecord = s.handleNulls(nullRecord) // we process nulls after writing + + readRecords, err = s.plugin.readAll(ctx, table) + if err != nil { + return fmt.Errorf("failed to sync: %w", err) + } + sortRecords(table, readRecords, "id") + + totalItems = TotalRows(readRecords) + if totalItems != 2*rowsPerRecord { + return fmt.Errorf("items expected after second insert: %d, got: %d", 2*rowsPerRecord, totalItems) + } + if diff := RecordsDiff(table.ToArrowSchema(), readRecords, []arrow.Record{normalRecord, nullRecord}); diff != "" { + return fmt.Errorf("record[0] differs: %s", diff) + } + return nil +} diff --git a/plugin-sdk/plugin/testing_write_migrate.go b/plugin-sdk/plugin/testing_write_migrate.go new file mode 100644 index 00000000..e01693da --- /dev/null +++ b/plugin-sdk/plugin/testing_write_migrate.go @@ -0,0 +1,364 @@ +package plugin + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" +) + +func tableUUIDSuffix() string { + return strings.ReplaceAll(uuid.NewString(), "-", "_")[:8] // use only first 8 chars +} + +// nolint:revive +func (s *WriterTestSuite) migrate(ctx context.Context, target *schema.Table, source *schema.Table, supportsSafeMigrate bool, writeOptionMigrateForce bool) error { + var rowsPerRecord = int(10) + if err := s.plugin.writeOne(ctx, &message.WriteMigrateTable{ + Table: source, + MigrateForce: writeOptionMigrateForce, + }); err != nil { + return fmt.Errorf("failed to create table: %w", err) + } + + records, err := s.plugin.readAll(ctx, source) + if err != nil { + return fmt.Errorf("failed to read initial records: %w", err) + } + initialItems := int(TotalRows(records)) + + sourceName := target.Name + syncTime := time.Now().UTC().Round(1 * time.Second) + opts := schema.GenTestDataOptions{ + SourceName: sourceName, + SyncTime: syncTime, + MaxRows: rowsPerRecord, + TimePrecision: s.genDatOptions.TimePrecision, + UseHomogeneousType: s.useHomogeneousTypes, + } + // Test Generator should be initialized with the current number of items in the destination + // this allows us to have multi-pass tests that ensure the migrations are stable + // create--> write --> migrate --> write -->migrate -->write-->migrate -->write + tg := schema.NewTestDataGenerator(uint64(initialItems)) + resource1 := tg.Generate(source, opts) + if err := s.plugin.writeOne(ctx, &message.WriteInsert{ + Record: resource1, + }); err != nil { + return fmt.Errorf("failed to insert first record: %w", err) + } + resource1 = s.handleNulls(resource1) // we process nulls after writing + + records, err = s.plugin.readAll(ctx, source) + if err != nil { + return fmt.Errorf("failed to sync: %w", err) + } + sortRecords(source, records, "id") + records = records[initialItems:] + + totalItems := TotalRows(records) + if totalItems != int64(rowsPerRecord) { + return fmt.Errorf("expected items: %d, got: %d", rowsPerRecord, totalItems) + } + + if diff := RecordsDiff(source.ToArrowSchema(), records, []arrow.Record{resource1}); diff != "" { + return fmt.Errorf("first record differs from expectation: %s", diff) + } + + if err := s.plugin.writeOne(ctx, &message.WriteMigrateTable{ + Table: target, + MigrateForce: writeOptionMigrateForce, + }); err != nil { + return fmt.Errorf("failed to create table: %w", err) + } + + resource2 := tg.Generate(target, opts) + if err := s.plugin.writeOne(ctx, &message.WriteInsert{ + Record: resource2, + }); err != nil { + return fmt.Errorf("failed to insert second record: %w", err) + } + resource2 = s.handleNulls(resource2) // we process nulls after writing + + records, err = s.plugin.readAll(ctx, target) + if err != nil { + return fmt.Errorf("failed to readAll: %w", err) + } + sortRecords(target, records, "id") + records = records[initialItems:] + lastRow := resource2.NewSlice(resource2.NumRows()-1, resource2.NumRows()) + // if force migration is not required, we don't expect any items to be dropped (so there should be 2 items) + if !writeOptionMigrateForce || supportsSafeMigrate { + if err := expectRows(target.ToArrowSchema(), records, 2*int64(rowsPerRecord), lastRow); err != nil { + if writeOptionMigrateForce && TotalRows(records) == int64(rowsPerRecord) { + // if force migration is required, we can also expect 1 item to be dropped + return expectRows(target.ToArrowSchema(), records, int64(rowsPerRecord), lastRow) + } + + return err + } + + return nil + } + + return expectRows(target.ToArrowSchema(), records, int64(rowsPerRecord), lastRow) +} + +// nolint:revive +func (s *WriterTestSuite) testMigrate( + ctx context.Context, + t *testing.T, + forceMigrate bool, +) { + suffix := "_safe" + if forceMigrate { + suffix = "_force" + } + t.Run("add_column"+suffix, func(t *testing.T) { + if !forceMigrate && !s.tests.SafeMigrations.AddColumn { + t.Skip("skipping test: add_column") + } + tableName := "cq_add_column" + suffix + "_" + tableUUIDSuffix() + source := &schema.Table{ + Name: tableName, + Columns: schema.ColumnList{ + {Name: "id", Type: arrow.PrimitiveTypes.Int64}, + {Name: "uuid", Type: types.ExtensionTypes.UUID}, + }, + } + + target := &schema.Table{ + Name: tableName, + Columns: schema.ColumnList{ + {Name: "id", Type: arrow.PrimitiveTypes.Int64}, + {Name: "uuid", Type: types.ExtensionTypes.UUID}, + {Name: "bool", Type: arrow.FixedWidthTypes.Boolean}, + }, + } + require.NoError(t, s.migrate(ctx, target, source, s.tests.SafeMigrations.AddColumn, forceMigrate)) + if !forceMigrate { + require.NoError(t, s.migrate(ctx, target, target, true, false)) + } + }) + + t.Run("add_column_not_null"+suffix, func(t *testing.T) { + if !forceMigrate && !s.tests.SafeMigrations.AddColumnNotNull { + t.Skip("skipping test: add_column_not_null") + } + tableName := "cq_add_column_not_null" + suffix + "_" + tableUUIDSuffix() + source := &schema.Table{ + Name: tableName, + Columns: schema.ColumnList{ + {Name: "id", Type: arrow.PrimitiveTypes.Int64}, + {Name: "uuid", Type: types.ExtensionTypes.UUID}, + }, + } + + target := &schema.Table{ + Name: tableName, + Columns: schema.ColumnList{ + {Name: "id", Type: arrow.PrimitiveTypes.Int64}, + {Name: "uuid", Type: types.ExtensionTypes.UUID}, + {Name: "bool", Type: arrow.FixedWidthTypes.Boolean, NotNull: true}, + }} + require.NoError(t, s.migrate(ctx, target, source, s.tests.SafeMigrations.AddColumnNotNull, forceMigrate)) + if !forceMigrate { + require.NoError(t, s.migrate(ctx, target, target, true, false)) + } + + }) + + t.Run("remove_column"+suffix, func(t *testing.T) { + if !forceMigrate && !s.tests.SafeMigrations.RemoveColumn { + t.Skip("skipping test: remove_column") + } + tableName := "cq_remove_column" + suffix + "_" + tableUUIDSuffix() + source := &schema.Table{ + Name: tableName, + Columns: schema.ColumnList{ + {Name: "id", Type: arrow.PrimitiveTypes.Int64}, + {Name: "uuid", Type: types.ExtensionTypes.UUID}, + {Name: "bool", Type: arrow.FixedWidthTypes.Boolean}, + }} + target := &schema.Table{ + Name: tableName, + Columns: schema.ColumnList{ + {Name: "id", Type: arrow.PrimitiveTypes.Int64}, + {Name: "uuid", Type: types.ExtensionTypes.UUID}, + }} + require.NoError(t, s.migrate(ctx, target, source, s.tests.SafeMigrations.RemoveColumn, forceMigrate)) + if !forceMigrate { + require.NoError(t, s.migrate(ctx, target, target, true, false)) + } + }) + + t.Run("remove_column_not_null"+suffix, func(t *testing.T) { + if !forceMigrate && !s.tests.SafeMigrations.RemoveColumnNotNull { + t.Skip("skipping test: remove_column_not_null") + } + tableName := "cq_remove_column_not_null" + suffix + "_" + tableUUIDSuffix() + source := &schema.Table{ + Name: tableName, + Columns: schema.ColumnList{ + {Name: "id", Type: arrow.PrimitiveTypes.Int64}, + {Name: "uuid", Type: types.ExtensionTypes.UUID}, + {Name: "bool", Type: arrow.FixedWidthTypes.Boolean, NotNull: true}, + }, + } + target := &schema.Table{ + Name: tableName, + Columns: schema.ColumnList{ + {Name: "id", Type: arrow.PrimitiveTypes.Int64}, + {Name: "uuid", Type: types.ExtensionTypes.UUID}, + }} + require.NoError(t, s.migrate(ctx, target, source, s.tests.SafeMigrations.RemoveColumnNotNull, forceMigrate)) + if !forceMigrate { + require.NoError(t, s.migrate(ctx, target, target, true, false)) + } + }) + + t.Run("change_column"+suffix, func(t *testing.T) { + if !forceMigrate && !s.tests.SafeMigrations.ChangeColumn { + t.Skip("skipping test: change_column") + } + tableName := "cq_change_column" + suffix + "_" + tableUUIDSuffix() + source := &schema.Table{ + Name: tableName, + Columns: schema.ColumnList{ + {Name: "id", Type: arrow.PrimitiveTypes.Int64}, + {Name: "uuid", Type: types.ExtensionTypes.UUID}, + {Name: "bool", Type: arrow.FixedWidthTypes.Boolean, NotNull: true}, + }} + target := &schema.Table{ + Name: tableName, + Columns: schema.ColumnList{ + {Name: "id", Type: arrow.PrimitiveTypes.Int64}, + {Name: "uuid", Type: types.ExtensionTypes.UUID}, + {Name: "bool", Type: arrow.BinaryTypes.String, NotNull: true}, + }} + require.NoError(t, s.migrate(ctx, target, source, s.tests.SafeMigrations.ChangeColumn, forceMigrate)) + if !forceMigrate { + require.NoError(t, s.migrate(ctx, target, target, true, false)) + } + }) + + t.Run("remove_unique_constraint_only"+suffix, func(t *testing.T) { + if s.tests.SkipSpecificMigrations.RemoveUniqueConstraint { + t.Skip("skipping test completely: remove_unique_constraint_only") + } + if !forceMigrate && !s.tests.SafeMigrations.RemoveUniqueConstraint { + t.Skip("skipping test: remove_unique_constraint_only") + } + tableName := "remove_unique_constraint_only" + suffix + "_" + tableUUIDSuffix() + source := &schema.Table{ + Name: tableName, + Columns: schema.ColumnList{ + {Name: "_cq_id", Type: types.ExtensionTypes.UUID, Unique: true}, + {Name: "id", Type: arrow.PrimitiveTypes.Int64}, + {Name: "uuid", Type: types.ExtensionTypes.UUID}, + {Name: "bool", Type: arrow.FixedWidthTypes.Boolean, NotNull: true}, + }} + target := &schema.Table{ + Name: tableName, + Columns: schema.ColumnList{ + {Name: "_cq_id", Type: types.ExtensionTypes.UUID}, + {Name: "id", Type: arrow.PrimitiveTypes.Int64}, + {Name: "uuid", Type: types.ExtensionTypes.UUID}, + {Name: "bool", Type: arrow.FixedWidthTypes.Boolean, NotNull: true}, + }} + require.NoError(t, s.migrate(ctx, target, source, s.tests.SafeMigrations.RemoveUniqueConstraint, forceMigrate)) + if !forceMigrate { + require.NoError(t, s.migrate(ctx, target, target, true, false)) + } + }) + + t.Run("move_to_cq_id_only"+suffix, func(t *testing.T) { + if s.tests.SkipSpecificMigrations.MovePKToCQOnly { + t.Skip("skipping test completely: move_to_cq_id_only") + } + if !forceMigrate && !s.tests.SafeMigrations.MovePKToCQOnly { + t.Skip("skipping test: move_to_cq_id_only") + } + tableName := "cq_move_to_cq_id_only" + suffix + "_" + tableUUIDSuffix() + source := &schema.Table{ + Name: tableName, + Columns: schema.ColumnList{ + {Name: "_cq_id", Type: types.ExtensionTypes.UUID, NotNull: true, Unique: true}, + {Name: "id", Type: arrow.PrimitiveTypes.Int64, PrimaryKey: true}, + {Name: "uuid", Type: types.ExtensionTypes.UUID}, + {Name: "bool", Type: arrow.FixedWidthTypes.Boolean, NotNull: true}, + }} + target := &schema.Table{ + Name: tableName, + Columns: schema.ColumnList{ + {Name: "_cq_id", Type: types.ExtensionTypes.UUID, NotNull: true, Unique: true, PrimaryKey: true}, + {Name: "id", Type: arrow.PrimitiveTypes.Int64}, + {Name: "uuid", Type: types.ExtensionTypes.UUID}, + {Name: "bool", Type: arrow.FixedWidthTypes.Boolean, NotNull: true}, + }} + require.NoError(t, s.migrate(ctx, target, source, s.tests.SafeMigrations.MovePKToCQOnly, forceMigrate)) + if !forceMigrate { + require.NoError(t, s.migrate(ctx, target, target, true, false)) + } + }) + t.Run("move_to_cq_id_only_adding_pkc"+suffix, func(t *testing.T) { + if s.tests.SkipSpecificMigrations.MovePKToCQOnly { + t.Skip("skipping test completely: move_to_cq_id_only_adding_pkc") + } + if !forceMigrate && !s.tests.SafeMigrations.MovePKToCQOnly { + t.Skip("skipping test: move_to_cq_id_only_adding_pkc") + } + tableName := "cq_move_to_cq_id_only_adding_pkc" + suffix + "_" + tableUUIDSuffix() + source := &schema.Table{ + Name: tableName, + Columns: schema.ColumnList{ + {Name: "_cq_id", Type: types.ExtensionTypes.UUID, NotNull: true, Unique: true}, + {Name: "id", Type: arrow.PrimitiveTypes.Int64, PrimaryKey: true}, + {Name: "uuid", Type: types.ExtensionTypes.UUID}, + {Name: "bool", Type: arrow.FixedWidthTypes.Boolean, NotNull: true, PrimaryKeyComponent: true}, + }} + target := &schema.Table{ + Name: tableName, + Columns: schema.ColumnList{ + {Name: "_cq_id", Type: types.ExtensionTypes.UUID, NotNull: true, Unique: true, PrimaryKey: true}, + {Name: "id", Type: arrow.PrimitiveTypes.Int64}, + {Name: "uuid", Type: types.ExtensionTypes.UUID}, + {Name: "bool", Type: arrow.FixedWidthTypes.Boolean, NotNull: true}, + }} + require.NoError(t, s.migrate(ctx, target, source, s.tests.SafeMigrations.MovePKToCQOnly, forceMigrate)) + if !forceMigrate { + require.NoError(t, s.migrate(ctx, target, target, true, false)) + } + }) + + t.Run("double_migration", func(t *testing.T) { + if forceMigrate { + t.Skip("double migration test has sense only for safe migrations") + } + tableName := "cq_double_migration_" + tableUUIDSuffix() + table := schema.TestTable(tableName, s.genDatOptions) + // s.migrate will perform create->write->migrate->write + require.NoError(t, s.migrate(ctx, table, table, true, false)) + }) +} + +func expectRows(sc *arrow.Schema, records []arrow.Record, expectTotal int64, expectedLast arrow.Record) error { + totalItems := TotalRows(records) + if totalItems != expectTotal { + return fmt.Errorf("expected %d items, got %d", expectTotal, totalItems) + } + lastRecord := records[len(records)-1] + lastRow := lastRecord.NewSlice(lastRecord.NumRows()-1, lastRecord.NumRows()) + if diff := RecordsDiff(sc, []arrow.Record{lastRow}, []arrow.Record{expectedLast}); diff != "" { + return fmt.Errorf("record #%d differs from expectation: %s", totalItems, diff) + } + return nil +} diff --git a/plugin-sdk/plugin/testing_write_upsert.go b/plugin-sdk/plugin/testing_write_upsert.go new file mode 100644 index 00000000..3296a89b --- /dev/null +++ b/plugin-sdk/plugin/testing_write_upsert.go @@ -0,0 +1,231 @@ +package plugin + +import ( + "context" + "fmt" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +func (s *WriterTestSuite) testUpsertBasic(ctx context.Context) error { + tableName := s.tableNameForTest("upsert_basic") + table := &schema.Table{ + Name: tableName, + Columns: []schema.Column{ + {Name: "id", Type: arrow.PrimitiveTypes.Int64, NotNull: true}, + {Name: "name", Type: arrow.BinaryTypes.String, PrimaryKey: true}, + }, + } + if err := s.plugin.writeOne(ctx, &message.WriteMigrateTable{ + Table: table, + }); err != nil { + return fmt.Errorf("failed to create table: %w", err) + } + + bldr := array.NewRecordBuilder(memory.DefaultAllocator, table.ToArrowSchema()) + bldr.Field(0).(*array.Int64Builder).Append(1) + bldr.Field(1).(*array.StringBuilder).Append("foo") + record := bldr.NewRecord() + + if err := s.plugin.writeOne(ctx, &message.WriteInsert{ + Record: record, + }); err != nil { + return fmt.Errorf("failed to insert record: %w", err) + } + record = s.handleNulls(record) // we process nulls after writing + + records, err := s.plugin.readAll(ctx, table) + if err != nil { + return fmt.Errorf("failed to readAll: %w", err) + } + totalItems := TotalRows(records) + if totalItems != 1 { + return fmt.Errorf("expected 1 item, got %d", totalItems) + } + + if err := s.plugin.writeOne(ctx, &message.WriteInsert{ + Record: record, + }); err != nil { + return fmt.Errorf("failed to insert record: %w", err) + } + + records, err = s.plugin.readAll(ctx, table) + if err != nil { + return fmt.Errorf("failed to sync: %w", err) + } + totalItems = TotalRows(records) + if totalItems != 1 { + return fmt.Errorf("expected 1 item, got %d", totalItems) + } + if diff := RecordsDiff(table.ToArrowSchema(), records, []arrow.Record{record}); diff != "" { + return fmt.Errorf("record differs: %s", diff) + } + return nil +} + +func (s *WriterTestSuite) testUpsertAll(ctx context.Context) error { + const rowsPerRecord = 10 + tableName := s.tableNameForTest("upsert_all") + table := schema.TestTable(tableName, s.genDatOptions) + table.Columns = append(table.Columns, schema.Column{Name: "name", Type: arrow.BinaryTypes.String, PrimaryKey: true}) + if err := s.plugin.writeOne(ctx, &message.WriteMigrateTable{ + Table: table, + }); err != nil { + return fmt.Errorf("failed to create table: %w", err) + } + + tg := schema.NewTestDataGenerator(0) + normalRecord := tg.Generate(table, schema.GenTestDataOptions{ + MaxRows: rowsPerRecord, + TimePrecision: s.genDatOptions.TimePrecision, + UseHomogeneousType: s.useHomogeneousTypes, + }) + if err := s.plugin.writeOne(ctx, &message.WriteInsert{ + Record: normalRecord, + }); err != nil { + return fmt.Errorf("failed to insert record: %w", err) + } + normalRecord = s.handleNulls(normalRecord) // we process nulls after writing + + records, err := s.plugin.readAll(ctx, table) + if err != nil { + return fmt.Errorf("failed to readAll: %w", err) + } + sortRecords(table, records, "id") + + totalItems := TotalRows(records) + if totalItems != rowsPerRecord { + return fmt.Errorf("expected items after initial insert: %d, got %d", rowsPerRecord, totalItems) + } + + if diff := RecordsDiff(table.ToArrowSchema(), records, []arrow.Record{normalRecord}); diff != "" { + return fmt.Errorf("record differs after insert: %s", diff) + } + + tg.Reset() + nullRecord := tg.Generate(table, schema.GenTestDataOptions{MaxRows: rowsPerRecord, TimePrecision: s.genDatOptions.TimePrecision, NullRows: true}) + if err := s.plugin.writeOne(ctx, &message.WriteInsert{ + Record: nullRecord, + }); err != nil { + return fmt.Errorf("failed to insert record: %w", err) + } + nullRecord = s.handleNulls(nullRecord) // we process nulls after writing + + records, err = s.plugin.readAll(ctx, table) + if err != nil { + return fmt.Errorf("failed to sync: %w", err) + } + sortRecords(table, records, "id") + + totalItems = TotalRows(records) + if totalItems != rowsPerRecord { + return fmt.Errorf("expected items after upsert: %d, got %d", rowsPerRecord, totalItems) + } + + if diff := RecordsDiff(table.ToArrowSchema(), records, []arrow.Record{nullRecord}); diff != "" { + return fmt.Errorf("record differs after upsert (columns should be null): %s", diff) + } + + return nil +} + +func (s *WriterTestSuite) testInsertDuplicatePK(ctx context.Context) error { + const rowsPerRecord = 10 + tableName := s.tableNameForTest("upsert_duplicate_pk") + table := &schema.Table{ + Name: tableName, + Columns: []schema.Column{ + {Name: "id", Type: arrow.PrimitiveTypes.Int64, NotNull: true, PrimaryKey: true}, + {Name: "name", Type: arrow.BinaryTypes.String}, + }, + } + if err := s.plugin.writeOne(ctx, &message.WriteMigrateTable{ + Table: table, + }); err != nil { + return fmt.Errorf("failed to create table: %w", err) + } + + // Create a multi-row record with a duplicate PK value, but different values for the other columns. + sc := table.ToArrowSchema() + var records []arrow.Record + for j := 0; j < rowsPerRecord; j++ { + bldr := array.NewRecordBuilder(memory.DefaultAllocator, sc) + bldr.Field(0).(*array.Int64Builder).Append(1) + bldr.Field(1).(*array.StringBuilder).Append("foo" + fmt.Sprint(j)) + records = append(records, bldr.NewRecord()) + bldr.Release() + } + + arrowTable := array.NewTableFromRecords(sc, records) + columns := make([]arrow.Array, sc.NumFields()) + for n := 0; n < sc.NumFields(); n++ { + concatenated, err := array.Concatenate(arrowTable.Column(n).Data().Chunks(), memory.DefaultAllocator) + if err != nil { + return fmt.Errorf("failed to concatenate arrays: %v", err) + } + columns[n] = concatenated + } + normalRecord := array.NewRecord(sc, columns, -1) + + // normalRecord + if err := s.plugin.writeOne(ctx, &message.WriteInsert{ + Record: normalRecord, + }); err != nil { + return fmt.Errorf("failed to insert record: %w", err) + } + + records, err := s.plugin.readAll(ctx, table) + if err != nil { + return fmt.Errorf("failed to readAll: %w", err) + } + sortRecords(table, records, "id") + + // We are only expecting a single row to be inserted. + totalItems := TotalRows(records) + if totalItems != 1 { + return fmt.Errorf("expected items after initial insert: %d, got %d", 1, totalItems) + } + + // Only the last row should be inserted. + lastRow, err := extractLastRowFromRecord(table, normalRecord) + if err != nil { + return fmt.Errorf("failed to extract last row from record: %w", err) + } + if diff := RecordsDiff(table.ToArrowSchema(), records, []arrow.Record{lastRow}); diff != "" { + return fmt.Errorf("record differs after insert: %s", diff) + } + + return nil +} + +func extractLastRowFromRecord(table *schema.Table, existingRecord arrow.Record) (arrow.Record, error) { + sc := table.ToArrowSchema() + bldr := array.NewRecordBuilder(memory.DefaultAllocator, sc) + for i, c := range table.Columns { + col := existingRecord.Column(i) + lastRow := int(existingRecord.NumRows()) - 1 + err := bldr.Field(i).AppendValueFromString(col.ValueStr(lastRow)) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal json `%v` for column %v: %v", col.ValueStr(lastRow), c.Name, err) + } + } + lastRecord := append([]arrow.Record{}, bldr.NewRecord()) + bldr.Release() + + arrowTable := array.NewTableFromRecords(sc, lastRecord) + columns := make([]arrow.Array, sc.NumFields()) + for n := 0; n < sc.NumFields(); n++ { + concatenated, err := array.Concatenate(arrowTable.Column(n).Data().Chunks(), memory.DefaultAllocator) + if err != nil { + return nil, fmt.Errorf("failed to concatenate arrays: %v", err) + } + columns[n] = concatenated + } + + return array.NewRecord(sc, columns, -1), nil +} diff --git a/plugin-sdk/plugin/validate.go b/plugin-sdk/plugin/validate.go new file mode 100644 index 00000000..86993e7b --- /dev/null +++ b/plugin-sdk/plugin/validate.go @@ -0,0 +1,47 @@ +package plugin + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/santhosh-tekuri/jsonschema/v5" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +func validateTables(tables schema.Tables) error { + if err := tables.ValidateDuplicateTables(); err != nil { + return fmt.Errorf("found duplicate tables in plugin: %w", err) + } + + if err := tables.ValidateDuplicateColumns(); err != nil { + return fmt.Errorf("found duplicate columns in plugin: %w", err) + } + + return nil +} + +func (p *Plugin) validate(ctx context.Context) error { + if p.skipTableValidation { + return nil + } + tables, err := p.client.Tables(ctx, TableOptions{Tables: []string{"*"}}) + // ErrNotImplemented means it's a destination only plugin + if err != nil && !errors.Is(err, ErrNotImplemented) { + return fmt.Errorf("failed to get tables: %w", err) + } + + return validateTables(tables) +} + +func JSONSchemaValidator(jsonSchema string) (*jsonschema.Schema, error) { + c := jsonschema.NewCompiler() + c.Draft = jsonschema.Draft2020 + c.AssertFormat = true + if err := c.AddResource("schema.json", strings.NewReader(jsonSchema)); err != nil { + return nil, err + } + return c.Compile("schema.json") +} diff --git a/plugin-sdk/premium/monitor.go b/plugin-sdk/premium/monitor.go new file mode 100644 index 00000000..7eafbf57 --- /dev/null +++ b/plugin-sdk/premium/monitor.go @@ -0,0 +1,108 @@ +package premium + +import ( + "context" + "errors" + "fmt" + "time" +) + +type ErrNoQuota struct { + team string +} + +func (e ErrNoQuota) Error() string { + return fmt.Sprintf("You have reached this plugin's usage limit for the month, please visit https://cloud.cloudquery.io/teams/%s/billing to upgrade your plan or increase the limit.", e.team) +} + +const DefaultQuotaCheckInterval = 30 * time.Second + +const DefaultMaxQuotaFailures = 10 // 5 minutes + +type quotaChecker struct { + qm QuotaMonitor + duration time.Duration + maxConsecutiveFailures int +} + +type QuotaCheckOption func(*quotaChecker) + +// WithQuotaCheckPeriod controls the time interval between quota checks +func WithQuotaCheckPeriod(duration time.Duration) QuotaCheckOption { + return func(m *quotaChecker) { + m.duration = duration + } +} + +// WithQuotaMaxConsecutiveFailures controls the number of consecutive failed quota checks before the context is cancelled +func WithQuotaMaxConsecutiveFailures(n int) QuotaCheckOption { + return func(m *quotaChecker) { + m.maxConsecutiveFailures = n + } +} + +// WithCancelOnQuotaExceeded monitors the quota usage at intervals defined by duration and cancels the context if the quota is exceeded +func WithCancelOnQuotaExceeded(ctx context.Context, qm QuotaMonitor, ops ...QuotaCheckOption) (context.Context, error) { + m := quotaChecker{ + qm: qm, + duration: DefaultQuotaCheckInterval, + maxConsecutiveFailures: DefaultMaxQuotaFailures, + } + for _, op := range ops { + op(&m) + } + + if err := m.checkInitialQuota(ctx); err != nil { + return ctx, err + } + + newCtx := m.startQuotaMonitor(ctx) + + return newCtx, nil +} + +func (qc quotaChecker) checkInitialQuota(ctx context.Context) error { + hasQuota, err := qc.qm.HasQuota(ctx) + if err != nil { + return err + } + + if !hasQuota { + return ErrNoQuota{team: qc.qm.TeamName()} + } + + return nil +} + +func (qc quotaChecker) startQuotaMonitor(ctx context.Context) context.Context { + newCtx, cancelWithCause := context.WithCancelCause(ctx) + go func() { + ticker := time.NewTicker(qc.duration) + consecutiveFailures := 0 + var hasQuotaErrors error + for { + select { + case <-newCtx.Done(): + return + case <-ticker.C: + hasQuota, err := qc.qm.HasQuota(newCtx) + if err != nil { + consecutiveFailures++ + hasQuotaErrors = errors.Join(hasQuotaErrors, err) + if consecutiveFailures >= qc.maxConsecutiveFailures { + cancelWithCause(hasQuotaErrors) + return + } + continue + } + consecutiveFailures = 0 + hasQuotaErrors = nil + if !hasQuota { + cancelWithCause(ErrNoQuota{team: qc.qm.TeamName()}) + return + } + } + } + }() + return newCtx +} diff --git a/plugin-sdk/premium/monitor_test.go b/plugin-sdk/premium/monitor_test.go new file mode 100644 index 00000000..0ce3876b --- /dev/null +++ b/plugin-sdk/premium/monitor_test.go @@ -0,0 +1,81 @@ +package premium + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +type quotaResponse struct { + hasQuota bool + err error +} + +func newFakeQuotaMonitor(hasQuota ...quotaResponse) *fakeQuotaMonitor { + return &fakeQuotaMonitor{responses: hasQuota} +} + +type fakeQuotaMonitor struct { + responses []quotaResponse + calls int +} + +func (f *fakeQuotaMonitor) HasQuota(_ context.Context) (bool, error) { + resp := f.responses[f.calls] + if f.calls < len(f.responses)-1 { + f.calls++ + } + return resp.hasQuota, resp.err +} + +func (*fakeQuotaMonitor) TeamName() string { + return "test" +} + +func TestWithCancelOnQuotaExceeded_NoInitialQuota(t *testing.T) { + ctx := context.Background() + + responses := []quotaResponse{ + {false, nil}, + } + _, err := WithCancelOnQuotaExceeded(ctx, newFakeQuotaMonitor(responses...)) + + require.Error(t, err) +} + +func TestWithCancelOnQuotaExceeded_NoQuota(t *testing.T) { + ctx := context.Background() + + responses := []quotaResponse{ + {true, nil}, + {false, nil}, + } + ctx, err := WithCancelOnQuotaExceeded(ctx, newFakeQuotaMonitor(responses...), WithQuotaCheckPeriod(1*time.Millisecond)) + require.NoError(t, err) + + <-ctx.Done() + cause := context.Cause(ctx) + require.ErrorIs(t, ErrNoQuota{team: "test"}, cause) +} + +func TestWithCancelOnQuotaCheckConsecutiveFailures(t *testing.T) { + ctx := context.Background() + + responses := []quotaResponse{ + {true, nil}, + {false, errors.New("test2")}, + {false, errors.New("test3")}, + } + ctx, err := WithCancelOnQuotaExceeded(ctx, + newFakeQuotaMonitor(responses...), + WithQuotaCheckPeriod(1*time.Millisecond), + WithQuotaMaxConsecutiveFailures(2), + ) + require.NoError(t, err) + <-ctx.Done() + cause := context.Cause(ctx) + require.Equal(t, "test2\ntest3", cause.Error()) +} diff --git a/plugin-sdk/premium/offline.go b/plugin-sdk/premium/offline.go new file mode 100644 index 00000000..06812c32 --- /dev/null +++ b/plugin-sdk/premium/offline.go @@ -0,0 +1,161 @@ +package premium + +import ( + "crypto/ed25519" + _ "embed" + "encoding/hex" + "encoding/json" + "errors" + "os" + "path/filepath" + "slices" + "strings" + "time" + + "github.com/rs/zerolog" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/plugin" +) + +type License struct { + LicensedTo string `json:"licensed_to"` // Customers name, e.g. "Acme Inc" + Plugins []string `json:"plugins,omitempty"` // List of plugins, each in the format //, e.g. "cloudquery/source/aws". Optional, if empty all plugins are allowed. + IssuedAt time.Time `json:"issued_at"` + ValidFrom time.Time `json:"valid_from"` + ExpiresAt time.Time `json:"expires_at"` +} + +type LicenseWrapper struct { + LicenseBytes []byte `json:"license"` + Signature string `json:"signature"` // crypto +} + +var ( + ErrInvalidLicenseSignature = errors.New("invalid license signature") + ErrLicenseNotValidYet = errors.New("license not valid yet") + ErrLicenseExpired = errors.New("license expired") + ErrLicenseNotApplicable = errors.New("license not applicable to this plugin") +) + +//go:embed offline.key +var publicKey string + +var timeFunc = time.Now + +func ValidateLicense(logger zerolog.Logger, meta plugin.Meta, licenseFileOrDirectory string) error { + fi, err := os.Stat(licenseFileOrDirectory) + if err != nil { + return err + } + if !fi.IsDir() { + return validateLicenseFile(logger, meta, licenseFileOrDirectory) + } + + found := false + var lastError error + err = filepath.WalkDir(licenseFileOrDirectory, func(path string, d os.DirEntry, err error) error { + if d.IsDir() { + if path == licenseFileOrDirectory { + return nil + } + return filepath.SkipDir + } + if err != nil { + return err + } + + if filepath.Ext(path) != ".cqlicense" { + return nil + } + + logger.Debug().Str("path", path).Msg("considering license file") + lastError = validateLicenseFile(logger, meta, path) + switch lastError { + case nil: + found = true + return filepath.SkipAll + case ErrLicenseNotApplicable: + return nil + default: + return lastError + } + }) + if err != nil { + return err + } + if found { + return nil + } + if lastError != nil { + return lastError + } + return errors.New("failed to validate license directory") +} + +func validateLicenseFile(logger zerolog.Logger, meta plugin.Meta, licenseFile string) error { + licenseContents, err := os.ReadFile(licenseFile) + if err != nil { + return err + } + + l, err := UnpackLicense(licenseContents) + if err != nil { + return err + } + + if len(l.Plugins) > 0 { + ref := strings.Join([]string{meta.Team, string(meta.Kind), meta.Name}, "/") + teamRef := meta.Team + "/*" + if !slices.Contains(l.Plugins, ref) && !slices.Contains(l.Plugins, teamRef) { + return ErrLicenseNotApplicable + } + } + + return l.IsValid(logger) +} + +func UnpackLicense(lic []byte) (*License, error) { + publicKeyBytes, err := hex.DecodeString(publicKey) + if err != nil { + return nil, err + } + + var lw LicenseWrapper + if err := json.Unmarshal(lic, &lw); err != nil { + return nil, err + } + + signatureBytes, err := hex.DecodeString(lw.Signature) + if err != nil { + return nil, err + } + + if !ed25519.Verify(publicKeyBytes, lw.LicenseBytes, signatureBytes) { + return nil, ErrInvalidLicenseSignature + } + + var l License + if err := json.Unmarshal(lw.LicenseBytes, &l); err != nil { + return nil, err + } + + return &l, nil +} + +func (l *License) IsValid(logger zerolog.Logger) error { + now := timeFunc().UTC() + if now.Before(l.ValidFrom) { + return ErrLicenseNotValidYet + } + if now.After(l.ExpiresAt) { + return ErrLicenseExpired + } + + msg := logger.Info() + if now.Add(15 * 24 * time.Hour).After(l.ExpiresAt) { + msg = logger.Warn() + } + + msg.Time("expires_at", l.ExpiresAt).Msgf("Offline license for %s loaded.", l.LicensedTo) + return nil +} diff --git a/plugin-sdk/premium/offline.key b/plugin-sdk/premium/offline.key new file mode 100644 index 00000000..5c93806a --- /dev/null +++ b/plugin-sdk/premium/offline.key @@ -0,0 +1 @@ +fd81a64351452e0ada99c05c7e44bdf104cc583eb3ed44bf5545fe82b2f0a615 \ No newline at end of file diff --git a/plugin-sdk/premium/offline_test.go b/plugin-sdk/premium/offline_test.go new file mode 100644 index 00000000..a7d4727a --- /dev/null +++ b/plugin-sdk/premium/offline_test.go @@ -0,0 +1,165 @@ +package premium + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/plugin" +) + +func TestUnpackLicense(t *testing.T) { + publicKey = "eacdff4866c8bc0d97de3c2d7668d0970c61aa16c3f12d6ba8083147ff92c9a6" + + t.Run("Success", func(t *testing.T) { + licData := `{"license":"eyJsaWNlbnNlZF90byI6IlVOTElDRU5TRUQgVEVTVCIsImlzc3VlZF9hdCI6IjIwMjMtMTItMjhUMTk6MDI6MjguODM4MzY3WiIsInZhbGlkX2Zyb20iOiIyMDIzLTEyLTI4VDE5OjAyOjI4LjgzODM2N1oiLCJleHBpcmVzX2F0IjoiMjAyMy0xMi0yOVQxOTowMjoyOC44MzgzNjdaIn0=","signature":"8687a858463764b052455b3c783d979d364b5fb653b86d88a7463e495480db62fdec7ae1a84d1e30dddee77eb769a0e498ecfc836538c53e410aeb1a0c04d102"}` + + l, err := UnpackLicense([]byte(licData)) + require.NoError(t, err) + require.Equal(t, "UNLICENSED TEST", l.LicensedTo) + require.Equal(t, l.ExpiresAt.Add(-24*time.Hour).Truncate(time.Hour), l.ValidFrom.Truncate(time.Hour)) + }) + t.Run("Fail", func(t *testing.T) { + licData := `{"license":"eyJsaWNlbnNlZF90byI6IlVOTElDRU5TRUQgVEVTVCIsImlzc3VlZF9hdCI6IjIwMjMtMTItMjhUMTk6MDI6MjguODM4MzY3WiIsInZhbGlkX2Zyb20iOiIyMDIzLTEyLTI4VDE5OjAyOjI4LjgzODM2N1oiLCJleHBpcmVzX2F0IjoiMjAyMy0xMi0yOVQxOTowMjoyOC44MzgzNjdaIn0=","signature":"9687a858463764b052455b3c783d979d364b5fb653b86d88a7463e495480db62fdec7ae1a84d1e30dddee77eb769a0e498ecfc836538c53e410aeb1a0c04d102"}` + l, err := UnpackLicense([]byte(licData)) + require.ErrorIs(t, err, ErrInvalidLicenseSignature) + require.Nil(t, l) + }) +} + +func TestValidateLicense(t *testing.T) { + publicKey = "eacdff4866c8bc0d97de3c2d7668d0970c61aa16c3f12d6ba8083147ff92c9a6" + licData := `{"license":"eyJsaWNlbnNlZF90byI6IlVOTElDRU5TRUQgVEVTVCIsImlzc3VlZF9hdCI6IjIwMjMtMTItMjhUMTk6MDI6MjguODM4MzY3WiIsInZhbGlkX2Zyb20iOiIyMDIzLTEyLTI4VDE5OjAyOjI4LjgzODM2N1oiLCJleHBpcmVzX2F0IjoiMjAyMy0xMi0yOVQxOTowMjoyOC44MzgzNjdaIn0=","signature":"8687a858463764b052455b3c783d979d364b5fb653b86d88a7463e495480db62fdec7ae1a84d1e30dddee77eb769a0e498ecfc836538c53e410aeb1a0c04d102"}` + validTime := time.Date(2023, 12, 29, 12, 0, 0, 0, time.UTC) + expiredTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC) + nopMeta := plugin.Meta{Team: "cloudquery", Kind: "source", Name: "test"} + + t.Run("SingleFile", func(t *testing.T) { + dir := t.TempDir() + f := filepath.Join(dir, "testlicense.cqlicense") + if err := os.WriteFile(f, []byte(licData), 0644); err != nil { + require.NoError(t, err) + } + + t.Run("Expired", licenseTest(f, nopMeta, expiredTime, ErrLicenseExpired)) + t.Run("Success", licenseTest(f, nopMeta, validTime, nil)) + }) + t.Run("Dir", func(t *testing.T) { + dir := t.TempDir() + f := filepath.Join(dir, "testlicense.cqlicense") + if err := os.WriteFile(f, []byte(licData), 0644); err != nil { + require.NoError(t, err) + } + t.Run("Expired", licenseTest(dir, nopMeta, expiredTime, ErrLicenseExpired)) + t.Run("Success", licenseTest(dir, nopMeta, validTime, nil)) + }) +} + +func TestValidateSpecificLicense(t *testing.T) { + publicKey = `de452e6028fe488f56ee0dfcf5b387ee773f03d24de66f00c40ec5b17085c549` + licData := `{"license":"eyJsaWNlbnNlZF90byI6IlVOTElDRU5TRUQgVEVTVCIsInBsdWdpbnMiOlsiY2xvdWRxdWVyeS9zb3VyY2UvdGVzdDEiLCJjbG91ZHF1ZXJ5L3NvdXJjZS90ZXN0MiJdLCJpc3N1ZWRfYXQiOiIyMDI0LTAxLTAyVDExOjEwOjA5LjE0OTYwNVoiLCJ2YWxpZF9mcm9tIjoiMjAyNC0wMS0wMlQxMToxMDowOS4xNDk2MDVaIiwiZXhwaXJlc19hdCI6IjIwMjQtMDEtMDNUMTE6MTA6MDkuMTQ5NjA1WiJ9","signature":"e5752577c2b2c5a8920b3277fd11504d9c6820e8acb22bc17ccda524857c1d9fc7534f39b9a122376069ad682a2b616a10d1cfae40a984fb57fee31f13a15302"}` + validTime := time.Date(2024, 1, 2, 12, 0, 0, 0, time.UTC) + expiredTime := time.Date(2024, 1, 3, 12, 0, 0, 0, time.UTC) + invalidMeta := plugin.Meta{Team: "cloudquery", Kind: "source", Name: "test"} + validMeta := plugin.Meta{Team: "cloudquery", Kind: "source", Name: "test1"} + + t.Run("SingleFile", func(t *testing.T) { + dir := t.TempDir() + f := filepath.Join(dir, "testlicense.cqlicense") + if err := os.WriteFile(f, []byte(licData), 0644); err != nil { + require.NoError(t, err) + } + + t.Run("Expired", licenseTest(f, validMeta, expiredTime, ErrLicenseExpired)) + t.Run("Success", licenseTest(f, validMeta, validTime, nil)) + t.Run("NotApplicable", licenseTest(f, invalidMeta, validTime, ErrLicenseNotApplicable)) + }) + t.Run("SingleDir", func(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "testlicense.cqlicense"), []byte(licData), 0644); err != nil { + require.NoError(t, err) + } + t.Run("Expired", licenseTest(dir, validMeta, expiredTime, ErrLicenseExpired)) + t.Run("Success", licenseTest(dir, validMeta, validTime, nil)) + t.Run("NotApplicable", licenseTest(dir, invalidMeta, validTime, ErrLicenseNotApplicable)) + }) +} + +func TestValidateSpecificLicenseMultiFile(t *testing.T) { + publicKey = `de452e6028fe488f56ee0dfcf5b387ee773f03d24de66f00c40ec5b17085c549` + licData1 := `{"license":"eyJsaWNlbnNlZF90byI6IlVOTElDRU5TRUQgVEVTVCIsInBsdWdpbnMiOlsiY2xvdWRxdWVyeS9zb3VyY2UvdGVzdDEiLCJjbG91ZHF1ZXJ5L3NvdXJjZS90ZXN0MiJdLCJpc3N1ZWRfYXQiOiIyMDI0LTAxLTAyVDExOjEwOjA5LjE0OTYwNVoiLCJ2YWxpZF9mcm9tIjoiMjAyNC0wMS0wMlQxMToxMDowOS4xNDk2MDVaIiwiZXhwaXJlc19hdCI6IjIwMjQtMDEtMDNUMTE6MTA6MDkuMTQ5NjA1WiJ9","signature":"e5752577c2b2c5a8920b3277fd11504d9c6820e8acb22bc17ccda524857c1d9fc7534f39b9a122376069ad682a2b616a10d1cfae40a984fb57fee31f13a15302"}` + licData3 := `{"license":"eyJsaWNlbnNlZF90byI6IlVOTElDRU5TRUQgVEVTVDMiLCJwbHVnaW5zIjpbImNsb3VkcXVlcnkvc291cmNlL3Rlc3QzIl0sImlzc3VlZF9hdCI6IjIwMjQtMDEtMDJUMTE6MjA6NTcuMzE2NDE0WiIsInZhbGlkX2Zyb20iOiIyMDI0LTAxLTAyVDExOjIwOjU3LjMxNjQxNFoiLCJleHBpcmVzX2F0IjoiMjAyNC0wMS0wM1QxMToyMDo1Ny4zMTY0MTRaIn0=","signature":"9be752d46010af84ec7295ede29915950dab13d4eca3b82b5645f793b39a03a6eef6bc653bee26e2a4f148b4d0fd54df6401059fda6104bc207f6dec2127850f"}` + + validTime := time.Date(2024, 1, 2, 12, 0, 0, 0, time.UTC) + expiredTime := time.Date(2024, 1, 3, 12, 0, 0, 0, time.UTC) + invalidMeta := plugin.Meta{Team: "cloudquery", Kind: "source", Name: "test"} + validMeta1 := plugin.Meta{Team: "cloudquery", Kind: "source", Name: "test1"} + validMeta3 := plugin.Meta{Team: "cloudquery", Kind: "source", Name: "test3"} + + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "testlicense1.cqlicense"), []byte(licData1), 0644); err != nil { + require.NoError(t, err) + } + if err := os.WriteFile(filepath.Join(dir, "testlicense3.cqlicense"), []byte(licData3), 0644); err != nil { + require.NoError(t, err) + } + + t.Run("Expired", licenseTest(dir, validMeta1, expiredTime, ErrLicenseExpired)) + t.Run("Success", licenseTest(dir, validMeta1, validTime, nil)) + t.Run("SuccessOther", licenseTest(dir, validMeta3, validTime, nil)) + t.Run("NotApplicable", licenseTest(dir, invalidMeta, validTime, ErrLicenseNotApplicable)) +} + +func TestValidateTeamLicense(t *testing.T) { + publicKey = `de452e6028fe488f56ee0dfcf5b387ee773f03d24de66f00c40ec5b17085c549` + licData := `{"license":"eyJsaWNlbnNlZF90byI6IlVOTElDRU5TRUQgVEVTVCIsInBsdWdpbnMiOlsidGVzdC10ZWFtLyoiXSwiaXNzdWVkX2F0IjoiMjAyNC0wMi0wNVQxNjozOTozMy4zMzkxMjZaIiwidmFsaWRfZnJvbSI6IjIwMjQtMDItMDVUMTY6Mzk6MzMuMzM5MTI2WiIsImV4cGlyZXNfYXQiOiIyMDI0LTAyLTA2VDE2OjM5OjMzLjMzOTEyNloifQ==","signature":"cba85dcbd48d909f92d6e84d1d56b47075484efb2a7db1c478fc09659bb498e2a761add3c743c2d9a50b82b29b1730600cd8f68d6571896ca7d08f3107751e07"}` + validTime := time.Date(2024, 2, 5, 18, 0, 0, 0, time.UTC) + expiredTime := time.Date(2024, 2, 6, 18, 0, 0, 0, time.UTC) + invalidMeta := plugin.Meta{Team: "cloudquery", Kind: "source", Name: "test"} + validMeta1 := plugin.Meta{Team: "test-team", Kind: "source", Name: "some-plugin"} + validMeta2 := plugin.Meta{Team: "test-team", Kind: "destination", Name: "some-plugin2"} + + t.Run("SingleFile", func(t *testing.T) { + dir := t.TempDir() + f := filepath.Join(dir, "testlicense.cqlicense") + if err := os.WriteFile(f, []byte(licData), 0644); err != nil { + require.NoError(t, err) + } + + t.Run("Expired1", licenseTest(f, validMeta1, expiredTime, ErrLicenseExpired)) + t.Run("Expired2", licenseTest(f, validMeta2, expiredTime, ErrLicenseExpired)) + t.Run("Success1", licenseTest(f, validMeta1, validTime, nil)) + t.Run("Success2", licenseTest(f, validMeta2, validTime, nil)) + t.Run("NotApplicable", licenseTest(f, invalidMeta, validTime, ErrLicenseNotApplicable)) + }) + t.Run("SingleDir", func(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "testlicense.cqlicense"), []byte(licData), 0644); err != nil { + require.NoError(t, err) + } + t.Run("Expired1", licenseTest(dir, validMeta1, expiredTime, ErrLicenseExpired)) + t.Run("Expired2", licenseTest(dir, validMeta2, expiredTime, ErrLicenseExpired)) + t.Run("Success1", licenseTest(dir, validMeta1, validTime, nil)) + t.Run("Success2", licenseTest(dir, validMeta2, validTime, nil)) + t.Run("NotApplicable", licenseTest(dir, invalidMeta, validTime, ErrLicenseNotApplicable)) + }) +} + +func licenseTest(inputPath string, meta plugin.Meta, timeIs time.Time, expectError error) func(t *testing.T) { + return func(t *testing.T) { + timeFunc = func() time.Time { + return timeIs + } + + err := ValidateLicense(zerolog.Nop(), meta, inputPath) + if expectError == nil { + require.NoError(t, err) + } else { + require.ErrorIs(t, err, expectError) + } + } +} diff --git a/plugin-sdk/premium/tables.go b/plugin-sdk/premium/tables.go new file mode 100644 index 00000000..64c8de88 --- /dev/null +++ b/plugin-sdk/premium/tables.go @@ -0,0 +1,25 @@ +package premium + +import "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + +// ContainsPaidTables returns true if any of the tables are paid +func ContainsPaidTables(tables schema.Tables) bool { + if tables == nil { + return false + } + for _, t := range tables { + if t.IsPaid || ContainsPaidTables(t.Relations) { + return true + } + } + return false +} + +// MakeAllTablesPaid sets all tables to paid (including relations) +func MakeAllTablesPaid(tables schema.Tables) schema.Tables { + for _, table := range tables { + table.IsPaid = true + MakeAllTablesPaid(table.Relations) + } + return tables +} diff --git a/plugin-sdk/premium/tables_test.go b/plugin-sdk/premium/tables_test.go new file mode 100644 index 00000000..6e68b212 --- /dev/null +++ b/plugin-sdk/premium/tables_test.go @@ -0,0 +1,51 @@ +package premium + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +func TestContainsPaidTables(t *testing.T) { + noPaidTables := schema.Tables{ + &schema.Table{Name: "table1", IsPaid: false}, + &schema.Table{Name: "table2", IsPaid: false}, + &schema.Table{Name: "table3", IsPaid: false}, + } + + paidTables := schema.Tables{ + &schema.Table{Name: "table1", IsPaid: false}, + &schema.Table{Name: "table2", IsPaid: true}, + &schema.Table{Name: "table3", IsPaid: false}, + } + + assert.False(t, ContainsPaidTables(noPaidTables), "no paid tables") + assert.True(t, ContainsPaidTables(paidTables), "paid tables") +} + +func TestMakeAllTablesPaid(t *testing.T) { + noPaidTables := schema.Tables{ + &schema.Table{Name: "table1", IsPaid: false}, + &schema.Table{Name: "table2", IsPaid: false}, + &schema.Table{Name: "table3", IsPaid: false}, + &schema.Table{Name: "table_with_relations", IsPaid: false, Relations: schema.Tables{ + &schema.Table{Name: "relation_table", IsPaid: false}, + }}, + } + + paidTables := MakeAllTablesPaid(noPaidTables) + + assert.Equal(t, 4, len(paidTables)) + assert.Equal(t, 5, len(paidTables.FlattenTables())) + assertAllArePaid(t, paidTables) +} + +func assertAllArePaid(t *testing.T, tables schema.Tables) { + t.Helper() + for _, table := range tables { + assert.True(t, table.IsPaid) + assertAllArePaid(t, table.Relations) + } +} diff --git a/plugin-sdk/premium/usage.go b/plugin-sdk/premium/usage.go new file mode 100644 index 00000000..005d30e1 --- /dev/null +++ b/plugin-sdk/premium/usage.go @@ -0,0 +1,578 @@ +package premium + +import ( + "context" + "errors" + "fmt" + "math/rand" + "net/http" + "os" + "strconv" + "sync" + "time" + + cqapi "github.com/cloudquery/cloudquery-api-go" + "github.com/cloudquery/cloudquery-api-go/auth" + "github.com/cloudquery/cloudquery-api-go/config" + "github.com/google/uuid" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/plugin" +) + +const ( + defaultAPIURL = "https://api.cloudquery.io" + defaultBatchLimit = 1000 + defaultMaxRetries = 5 + defaultMaxWaitTime = 60 * time.Second + defaultMinTimeBetweenFlushes = 10 * time.Second + defaultMaxTimeBetweenFlushes = 30 * time.Second +) + +const ( + UsageIncreaseMethodUnset = iota + UsageIncreaseMethodTotal + UsageIncreaseMethodBreakdown +) + +const ( + BatchLimitHeader = "x-cq-batch-limit" + MinimumUpdateIntervalHeader = "x-cq-minimum-update-interval" + MaximumUpdateIntervalHeader = "x-cq-maximum-update-interval" +) + +type TokenClient interface { + GetToken() (auth.Token, error) + GetTokenType() auth.TokenType +} + +type QuotaMonitor interface { + // TeamName returns the team name + TeamName() string + // HasQuota returns true if the quota has not been exceeded + HasQuota(context.Context) (bool, error) +} + +type UsageClient interface { + QuotaMonitor + // Increase updates the usage by the given number of rows + Increase(uint32) error + // IncreaseForTable updates the usage of a table by the given number of rows + IncreaseForTable(string, uint32) error + // Close flushes any remaining rows and closes the quota service + Close() error +} + +type UsageClientOptions func(updater *BatchUpdater) + +// WithBatchLimit sets the maximum number of rows to update in a single request +func WithBatchLimit(batchLimit uint32) UsageClientOptions { + return func(updater *BatchUpdater) { + updater.batchLimit = batchLimit + } +} + +// WithMaxTimeBetweenFlushes sets the flush duration - the time at which an update will be triggered even if the batch limit is not reached +func WithMaxTimeBetweenFlushes(maxTimeBetweenFlushes time.Duration) UsageClientOptions { + return func(updater *BatchUpdater) { + updater.maxTimeBetweenFlushes = maxTimeBetweenFlushes + updater.flushDuration.Reset(maxTimeBetweenFlushes) + } +} + +// WithMinTimeBetweenFlushes sets the minimum time between updates +func WithMinTimeBetweenFlushes(minTimeBetweenFlushes time.Duration) UsageClientOptions { + return func(updater *BatchUpdater) { + updater.minTimeBetweenFlushes = minTimeBetweenFlushes + } +} + +// WithMaxRetries sets the maximum number of retries to update the usage in case of an API error +func WithMaxRetries(maxRetries int) UsageClientOptions { + return func(updater *BatchUpdater) { + updater.maxRetries = maxRetries + } +} + +// WithMaxWaitTime sets the maximum time to wait before retrying a failed update +func WithMaxWaitTime(maxWaitTime time.Duration) UsageClientOptions { + return func(updater *BatchUpdater) { + updater.maxWaitTime = maxWaitTime + } +} + +// WithLogger sets the logger to use - defaults to a no-op logger +func WithLogger(logger zerolog.Logger) UsageClientOptions { + return func(updater *BatchUpdater) { + updater.logger = logger + } +} + +// WithURL sets the API URL to use - defaults to https://api.cloudquery.io +func WithURL(url string) UsageClientOptions { + return func(updater *BatchUpdater) { + updater.url = url + } +} + +// withTeamName sets the team name to use - defaults to the team name from the configuration +func withTeamName(teamName cqapi.TeamName) UsageClientOptions { + return func(updater *BatchUpdater) { + updater.teamName = teamName + } +} + +// WithAPIClient sets the API client to use - defaults to a client using a bearer token generated from the refresh token stored in the configuration +func WithAPIClient(apiClient *cqapi.ClientWithResponses) UsageClientOptions { + return func(updater *BatchUpdater) { + updater.apiClient = apiClient + } +} + +// withTokenClient sets the token client to use - defaults to auth.NewTokenClient(). Used in tests to mock the token client +func withTokenClient(tokenClient TokenClient) UsageClientOptions { + return func(updater *BatchUpdater) { + updater.tokenClient = tokenClient + } +} + +var ( + _ UsageClient = (*BatchUpdater)(nil) + _ UsageClient = (*NoOpUsageClient)(nil) +) + +type BatchUpdater struct { + logger zerolog.Logger + url string + apiClient *cqapi.ClientWithResponses + tokenClient TokenClient + + // Plugin details + teamName cqapi.TeamName + pluginMeta plugin.Meta + + // Configuration + batchLimit uint32 + maxRetries int + maxWaitTime time.Duration + minTimeBetweenFlushes time.Duration + maxTimeBetweenFlushes time.Duration + + // State + sync.Mutex + flushDuration *time.Ticker + rows uint32 + tables map[string]uint32 + lastUpdateTime time.Time + triggerUpdate chan struct{} + done chan struct{} + closeError chan error + isClosed bool + usageIncreaseMethod int +} + +func NewUsageClient(meta plugin.Meta, ops ...UsageClientOptions) (UsageClient, error) { + u := &BatchUpdater{ + logger: zerolog.Nop(), + url: defaultAPIURL, + + pluginMeta: meta, + + batchLimit: defaultBatchLimit, + minTimeBetweenFlushes: defaultMinTimeBetweenFlushes, + maxTimeBetweenFlushes: defaultMaxTimeBetweenFlushes, + maxRetries: defaultMaxRetries, + maxWaitTime: defaultMaxWaitTime, + flushDuration: time.NewTicker(defaultMaxTimeBetweenFlushes), + triggerUpdate: make(chan struct{}), + done: make(chan struct{}), + closeError: make(chan error), + + tables: map[string]uint32{}, + } + for _, op := range ops { + op(u) + } + + if meta.SkipUsageClient { + u.logger.Debug().Msg("Disabling usage client") + return &NoOpUsageClient{ + TeamNameValue: u.teamName, + }, nil + } + + if u.tokenClient == nil { + u.tokenClient = auth.NewTokenClient() + } + + // Fail early if the token is not set + if _, err := u.tokenClient.GetToken(); err != nil { + return nil, err + } + + // Create a default api client if none was provided + if u.apiClient == nil { + ac, err := cqapi.NewClientWithResponses(u.url, cqapi.WithRequestEditorFn(func(_ context.Context, req *http.Request) error { + token, err := u.tokenClient.GetToken() + if err != nil { + return fmt.Errorf("failed to get token: %w", err) + } + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + return nil + })) + if err != nil { + return nil, fmt.Errorf("failed to create api client: %w", err) + } + u.apiClient = ac + } + + // Set team name from configuration if not provided + if u.teamName == "" { + teamName, err := u.getTeamNameByTokenType(u.tokenClient.GetTokenType()) + if err != nil { + return nil, fmt.Errorf("failed to get team name: %w", err) + } + u.teamName = teamName + } + + u.backgroundUpdater() + + return u, nil +} + +func (u *BatchUpdater) Increase(rows uint32) error { + if u.usageIncreaseMethod == UsageIncreaseMethodBreakdown { + return fmt.Errorf("mixing usage increase methods is not allowed, use IncreaseForTable instead") + } + + if rows <= 0 { + return fmt.Errorf("rows must be greater than zero got %d", rows) + } + + if u.isClosed { + return fmt.Errorf("usage updater is closed") + } + + u.Lock() + defer u.Unlock() + + if u.usageIncreaseMethod == UsageIncreaseMethodUnset { + u.usageIncreaseMethod = UsageIncreaseMethodTotal + } + u.rows += rows + + // Trigger an update unless an update is already in process + select { + case u.triggerUpdate <- struct{}{}: + default: + } + + return nil +} + +func (u *BatchUpdater) IncreaseForTable(table string, rows uint32) error { + if u.usageIncreaseMethod == UsageIncreaseMethodTotal { + return fmt.Errorf("mixing usage increase methods is not allowed, use Increase instead") + } + + if rows <= 0 { + return fmt.Errorf("rows must be greater than zero got %d", rows) + } + + if u.isClosed { + return fmt.Errorf("usage updater is closed") + } + + u.Lock() + defer u.Unlock() + + if u.usageIncreaseMethod == UsageIncreaseMethodUnset { + u.usageIncreaseMethod = UsageIncreaseMethodBreakdown + } + + u.tables[table] += rows + u.rows += rows + + // Trigger an update unless an update is already in process + select { + case u.triggerUpdate <- struct{}{}: + default: + } + + return nil +} + +func (u *BatchUpdater) TeamName() string { + return u.teamName +} + +func (u *BatchUpdater) HasQuota(ctx context.Context) (bool, error) { + u.logger.Debug().Str("url", u.url).Str("team", u.teamName).Str("pluginTeam", u.pluginMeta.Team).Str("pluginKind", string(u.pluginMeta.Kind)).Str("pluginName", u.pluginMeta.Name).Msg("checking quota") + usage, err := u.apiClient.GetTeamPluginUsageWithResponse(ctx, u.teamName, u.pluginMeta.Team, u.pluginMeta.Kind, u.pluginMeta.Name) + if err != nil { + return false, fmt.Errorf("failed to get usage: %w", err) + } + if usage.StatusCode() != http.StatusOK { + return false, fmt.Errorf("failed to get usage: %s", usage.Status()) + } + + hasQuota := usage.JSON200.RemainingRows == nil || *usage.JSON200.RemainingRows > 0 + return hasQuota, nil +} + +func (u *BatchUpdater) Close() error { + u.isClosed = true + + close(u.done) + + return <-u.closeError +} + +func (u *BatchUpdater) getTableUsage() (usage []cqapi.UsageIncreaseTablesInner, total uint32) { + u.Lock() + defer u.Unlock() + + for key, value := range u.tables { + usage = append(usage, cqapi.UsageIncreaseTablesInner{ + Name: key, + Rows: int(value), + }) + } + + return usage, u.rows +} + +func (u *BatchUpdater) subtractTableUsage(usage []cqapi.UsageIncreaseTablesInner, total uint32) { + u.Lock() + defer u.Unlock() + + for _, table := range usage { + u.tables[table.Name] -= uint32(table.Rows) + } + + u.rows -= total +} + +func (u *BatchUpdater) backgroundUpdater() { + ctx := context.Background() + started := make(chan struct{}) + + go func() { + started <- struct{}{} + for { + select { + case <-u.triggerUpdate: + if time.Since(u.lastUpdateTime) < u.minTimeBetweenFlushes { + // Not enough time since last update + continue + } + + tables, totals := u.getTableUsage() + + if totals < u.batchLimit { + // Not enough rows to update + continue + } + + if err := u.updateUsageWithRetryAndBackoff(ctx, totals, tables); err != nil { + log.Warn().Err(err).Msg("failed to update usage") + continue + } + u.subtractTableUsage(tables, totals) + + case <-u.flushDuration.C: + if time.Since(u.lastUpdateTime) < u.minTimeBetweenFlushes { + // Not enough time since last update + continue + } + + tables, totals := u.getTableUsage() + + if totals == 0 { + continue + } + + if err := u.updateUsageWithRetryAndBackoff(ctx, totals, tables); err != nil { + log.Warn().Err(err).Msg("failed to update usage") + continue + } + u.subtractTableUsage(tables, totals) + + case <-u.done: + tables, totals := u.getTableUsage() + if totals != 0 { + if err := u.updateUsageWithRetryAndBackoff(ctx, totals, tables); err != nil { + u.closeError <- err + return + } + u.subtractTableUsage(tables, totals) + } + u.closeError <- nil + return + } + } + }() + <-started +} + +func (u *BatchUpdater) updateUsageWithRetryAndBackoff(ctx context.Context, rows uint32, tables []cqapi.UsageIncreaseTablesInner) error { + for retry := 0; retry < u.maxRetries; retry++ { + u.logger.Debug().Str("url", u.url).Int("try", retry).Int("max_retries", u.maxRetries).Uint32("rows", rows).Msg("updating usage") + queryStartTime := time.Now() + + payload := cqapi.IncreaseTeamPluginUsageJSONRequestBody{ + RequestId: uuid.New(), + PluginTeam: u.pluginMeta.Team, + PluginKind: u.pluginMeta.Kind, + PluginName: u.pluginMeta.Name, + Rows: int(rows), + } + + if len(tables) > 0 { + payload.Tables = &tables + } + + resp, err := u.apiClient.IncreaseTeamPluginUsageWithResponse(ctx, u.teamName, payload) + if err != nil { + return fmt.Errorf("failed to update usage: %w", err) + } + if resp.StatusCode() >= 200 && resp.StatusCode() < 300 { + u.logger.Debug().Str("url", u.url).Int("try", retry).Int("status_code", resp.StatusCode()).Uint32("rows", rows).Msg("usage updated") + u.lastUpdateTime = time.Now().UTC() + if resp.HTTPResponse != nil { + u.updateConfigurationFromHeaders(resp.HTTPResponse.Header) + } + return nil + } + + retryDuration, err := u.calculateRetryDuration(resp.StatusCode(), resp.HTTPResponse.Header, queryStartTime, retry) + if err != nil { + return fmt.Errorf("failed to calculate retry duration: %w", err) + } + if retryDuration > 0 { + time.Sleep(retryDuration) + } + } + return fmt.Errorf("failed to update usage: max retries exceeded") +} + +// updateConfigurationFromHeaders updates the configuration based on the headers returned by the API +func (u *BatchUpdater) updateConfigurationFromHeaders(header http.Header) { + if headerValue := header.Get(BatchLimitHeader); headerValue != "" { + if newBatchLimit, err := strconv.ParseUint(headerValue, 10, 32); err != nil { + u.logger.Warn().Err(err).Str(BatchLimitHeader, headerValue).Msg("failed to parse batch limit") + } else { + u.batchLimit = uint32(newBatchLimit) + } + } + + if headerValue := header.Get(MinimumUpdateIntervalHeader); headerValue != "" { + if newInterval, err := strconv.ParseInt(headerValue, 10, 32); err != nil { + u.logger.Warn().Err(err).Str(MinimumUpdateIntervalHeader, headerValue).Msg("failed to parse minimum update interval") + } else { + u.minTimeBetweenFlushes = time.Duration(newInterval) * time.Second + } + } + + if headerValue := header.Get(MaximumUpdateIntervalHeader); headerValue != "" { + if newInterval, err := strconv.ParseInt(headerValue, 10, 32); err != nil { + u.logger.Warn().Err(err).Str(MaximumUpdateIntervalHeader, headerValue).Msg("failed to parse maximum update interval") + } else { + newMaxTimeBetweenFlushes := time.Duration(newInterval) * time.Second + if u.maxTimeBetweenFlushes != newMaxTimeBetweenFlushes { + u.maxTimeBetweenFlushes = newMaxTimeBetweenFlushes + u.flushDuration.Reset(u.maxTimeBetweenFlushes) + } + } + } +} + +// calculateRetryDuration calculates the duration to sleep relative to the query start time before retrying an update +func (u *BatchUpdater) calculateRetryDuration(statusCode int, headers http.Header, queryStartTime time.Time, retry int) (time.Duration, error) { + if !retryableStatusCode(statusCode) { + return 0, fmt.Errorf("non-retryable status code: %d", statusCode) + } + + // Check if we have a retry-after header + retryAfter := headers.Get("Retry-After") + if retryAfter != "" { + retryDelay, err := time.ParseDuration(retryAfter + "s") + if err != nil { + return 0, fmt.Errorf("failed to parse retry-after header: %w", err) + } + return retryDelay, nil + } + + // Calculate exponential backoff + baseRetry := min(time.Duration(1<`", err) + } else if err != nil { + return "", fmt.Errorf("failed to get team name from config: %w", err) + } + if teamName == "" { + return "", fmt.Errorf("team name not set. Hint: use `cloudquery switch `") + } + return teamName, nil + case auth.APIKey: + resp, err := u.apiClient.ListTeamsWithResponse(context.Background(), &cqapi.ListTeamsParams{}) + if err != nil { + return "", fmt.Errorf("failed to list teams for API key: %w", err) + } + if resp.StatusCode() != http.StatusOK { + return "", fmt.Errorf("failed to list teams for API key, status code: %s", resp.Status()) + } + if len(resp.JSON200.Items) != 1 { + return "", fmt.Errorf("expected to find exactly one team for API key, found %d", len(resp.JSON200.Items)) + } + return resp.JSON200.Items[0].Name, nil + default: + team := os.Getenv("_CQ_TEAM_NAME") + if team == "" { + switch tokenType { + case auth.SyncRunAPIKey, auth.SyncTestConnectionAPIKey: + return "", fmt.Errorf("_CQ_TEAM_NAME environment variable not set") + } + return "", fmt.Errorf("unsupported token type: %v", tokenType) + } + return team, nil + } +} + +type NoOpUsageClient struct { + TeamNameValue string +} + +func (n *NoOpUsageClient) TeamName() string { + return n.TeamNameValue +} + +func (NoOpUsageClient) HasQuota(_ context.Context) (bool, error) { + return true, nil +} + +func (NoOpUsageClient) Increase(_ uint32) error { + return nil +} + +func (NoOpUsageClient) IncreaseForTable(_ string, _ uint32) error { + return nil +} + +func (NoOpUsageClient) Close() error { + return nil +} diff --git a/plugin-sdk/premium/usage_test.go b/plugin-sdk/premium/usage_test.go new file mode 100644 index 00000000..fb5dba06 --- /dev/null +++ b/plugin-sdk/premium/usage_test.go @@ -0,0 +1,829 @@ +package premium + +import ( + "context" + "encoding/json" + "fmt" + "math" + "net/http" + "net/http/httptest" + "strconv" + "sync" + "testing" + "time" + + cqapi "github.com/cloudquery/cloudquery-api-go" + "github.com/cloudquery/cloudquery-api-go/auth" + "github.com/cloudquery/cloudquery-api-go/config" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/plugin" +) + +type MockTokenClient struct { + tokenType auth.TokenType +} + +func (*MockTokenClient) GetToken() (auth.Token, error) { + return auth.Token{}, nil +} + +func (c *MockTokenClient) GetTokenType() auth.TokenType { + return c.tokenType +} + +func newMockTokenClient(tokenType auth.TokenType) *MockTokenClient { + return &MockTokenClient{tokenType: tokenType} +} + +func TestUsageService_NewUsageClient_Defaults(t *testing.T) { + err := config.SetConfigHome(t.TempDir()) + require.NoError(t, err) + + err = config.SetValue("team", "config-team") + require.NoError(t, err) + + uc, err := NewUsageClient( + plugin.Meta{ + Team: "plugin-team", + Kind: cqapi.PluginKindSource, + Name: "vault", + }, + withTokenClient(newMockTokenClient(auth.BearerToken)), + ) + require.NoError(t, err) + + bu := uc.(*BatchUpdater) + + assert.NotNil(t, bu.apiClient) + assert.Equal(t, "config-team", bu.teamName) + assert.Equal(t, zerolog.Nop(), bu.logger) + assert.Equal(t, 5, bu.maxRetries) + assert.Equal(t, 60*time.Second, bu.maxWaitTime) + assert.Equal(t, 30*time.Second, bu.maxTimeBetweenFlushes) +} + +func TestUsageService_NewUsageClient_Override(t *testing.T) { + ac, err := cqapi.NewClientWithResponses("http://localhost") + require.NoError(t, err) + + logger := zerolog.New(zerolog.NewTestWriter(t)) + + uc, err := NewUsageClient( + plugin.Meta{ + Team: "plugin-team", + Kind: cqapi.PluginKindSource, + Name: "vault", + }, + WithLogger(logger), + WithAPIClient(ac), + withTeamName("override-team-name"), + WithMaxRetries(10), + WithMaxWaitTime(120*time.Second), + WithMaxTimeBetweenFlushes(10*time.Second), + withTokenClient(newMockTokenClient(auth.BearerToken)), + ) + require.NoError(t, err) + + bu := uc.(*BatchUpdater) + + assert.Equal(t, ac, bu.apiClient) + assert.Equal(t, "override-team-name", bu.teamName) + assert.Equal(t, logger, bu.logger) + assert.Equal(t, 10, bu.maxRetries) + assert.Equal(t, 120*time.Second, bu.maxWaitTime) + assert.Equal(t, 10*time.Second, bu.maxTimeBetweenFlushes) +} + +func TestUsageService_HasQuota_NoRowsRemaining(t *testing.T) { + ctx := context.Background() + + s := createTestServerWithRemainingRows(t, 0) + defer s.server.Close() + + apiClient, err := cqapi.NewClientWithResponses(s.server.URL) + require.NoError(t, err) + + usageClient := newClient(t, apiClient, WithBatchLimit(0)) + + hasQuota, err := usageClient.HasQuota(ctx) + require.NoError(t, err) + + assert.False(t, hasQuota, "should not have quota") +} + +func TestUsageService_HasQuota_WithRowsRemaining(t *testing.T) { + ctx := context.Background() + + s := createTestServerWithRemainingRows(t, 100) + defer s.server.Close() + + apiClient, err := cqapi.NewClientWithResponses(s.server.URL) + require.NoError(t, err) + + usageClient := newClient(t, apiClient, WithBatchLimit(0)) + + hasQuota, err := usageClient.HasQuota(ctx) + require.NoError(t, err) + + assert.True(t, hasQuota, "should have quota") +} + +func TestUsageService_Increase_ZeroBatchSize(t *testing.T) { + s := createTestServer(t) + defer s.server.Close() + + apiClient, err := cqapi.NewClientWithResponses(s.server.URL) + require.NoError(t, err) + + usageClient := newClient(t, apiClient, WithBatchLimit(0)) + + for i := 0; i < 10000; i++ { + err = usageClient.Increase(1) + require.NoError(t, err) + } + + err = usageClient.Close() + require.NoError(t, err) + + assert.Equal(t, 10000, s.sumOfUpdates(), "total should equal number of updated rows") +} + +func TestUsageService_IncreaseForTable_ZeroBatchSize(t *testing.T) { + s := createTestServer(t) + defer s.server.Close() + + apiClient, err := cqapi.NewClientWithResponses(s.server.URL) + require.NoError(t, err) + + usageClient := newClient(t, apiClient, WithBatchLimit(0)) + + tables := 3 + rows := 9999 + for i := 0; i < rows; i++ { + table := "table:" + strconv.Itoa(i%tables) + err = usageClient.IncreaseForTable(table, 1) + require.NoError(t, err) + } + + err = usageClient.Close() + require.NoError(t, err) + + assert.Equal(t, rows, s.sumOfUpdates(), "total should equal number of updated rows") + assert.Equal(t, rows, s.sumOfTableUpdates(), "breakdown over tables should equal number of updated rows") +} + +func TestUsageService_Increase_WithBatchSize(t *testing.T) { + batchSize := 2000 + + s := createTestServer(t) + defer s.server.Close() + + apiClient, err := cqapi.NewClientWithResponses(s.server.URL) + require.NoError(t, err) + + usageClient := newClient(t, apiClient, WithBatchLimit(uint32(batchSize))) + + for i := 0; i < 10000; i++ { + err = usageClient.Increase(1) + require.NoError(t, err) + } + err = usageClient.Close() + require.NoError(t, err) + + assert.Equal(t, 10000, s.sumOfUpdates(), "total should equal number of updated rows") + assert.True(t, true, s.minExcludingClose() > batchSize, "minimum should be greater than batch size") +} + +func TestUsageService_IncreaseForTable_WithBatchSize(t *testing.T) { + batchSize := 2000 + + s := createTestServer(t) + defer s.server.Close() + + apiClient, err := cqapi.NewClientWithResponses(s.server.URL) + require.NoError(t, err) + + usageClient := newClient(t, apiClient, WithBatchLimit(uint32(batchSize))) + + tables := 3 + rows := 9999 + for i := 0; i < rows; i++ { + table := "table:" + strconv.Itoa(i%tables) + err = usageClient.IncreaseForTable(table, 1) + require.NoError(t, err) + } + err = usageClient.Close() + require.NoError(t, err) + + assert.Equal(t, rows, s.sumOfUpdates(), "total should equal number of updated rows") + assert.Equal(t, rows, s.sumOfTableUpdates(), "breakdown over tables should equal number of updated rows") + assert.True(t, true, s.minExcludingClose() > batchSize, "minimum should be greater than batch size") +} + +func TestUsageService_Increase_WithFlushDuration(t *testing.T) { + batchSize := 2000 + + s := createTestServer(t) + defer s.server.Close() + + apiClient, err := cqapi.NewClientWithResponses(s.server.URL) + require.NoError(t, err) + + usageClient := newClient(t, apiClient, WithBatchLimit(uint32(batchSize)), WithMaxTimeBetweenFlushes(1*time.Millisecond), WithMinTimeBetweenFlushes(0*time.Millisecond)) + + for i := 0; i < 10; i++ { + err = usageClient.Increase(10) + require.NoError(t, err) + time.Sleep(5 * time.Millisecond) + } + err = usageClient.Close() + require.NoError(t, err) + + assert.Equal(t, 100, s.sumOfUpdates(), "total should equal number of updated rows") + assert.True(t, s.minExcludingClose() < batchSize, "we should see updates less than batchsize if ticker is firing") +} + +func TestUsageService_IncreaseForTable_WithFlushDuration(t *testing.T) { + batchSize := 2000 + + s := createTestServer(t) + defer s.server.Close() + + apiClient, err := cqapi.NewClientWithResponses(s.server.URL) + require.NoError(t, err) + + usageClient := newClient(t, apiClient, WithBatchLimit(uint32(batchSize)), WithMaxTimeBetweenFlushes(1*time.Millisecond), WithMinTimeBetweenFlushes(0*time.Millisecond)) + + tables := 3 + rows := 30 + for i := 0; i < rows; i++ { + table := "table:" + strconv.Itoa(i%tables) + err = usageClient.IncreaseForTable(table, 1) + require.NoError(t, err) + time.Sleep(5 * time.Millisecond) + } + err = usageClient.Close() + require.NoError(t, err) + + assert.Equal(t, rows, s.sumOfUpdates(), "total should equal number of updated rows") + assert.Equal(t, rows, s.sumOfTableUpdates(), "breakdown over tables should equal number of updated rows") + assert.True(t, s.minExcludingClose() < batchSize, "we should see updates less than batchsize if ticker is firing") +} + +func TestUsageService_Increase_WithMinimumUpdateDuration(t *testing.T) { + s := createTestServer(t) + defer s.server.Close() + + apiClient, err := cqapi.NewClientWithResponses(s.server.URL) + require.NoError(t, err) + + usageClient := newClient(t, apiClient, WithBatchLimit(0), WithMinTimeBetweenFlushes(30*time.Second)) + + for i := 0; i < 10000; i++ { + err = usageClient.Increase(1) + require.NoError(t, err) + } + err = usageClient.Close() + require.NoError(t, err) + + assert.Equal(t, 10000, s.sumOfUpdates(), "total should equal number of updated rows") + assert.Equal(t, 2, s.numberOfUpdates(), "should only update first time and on close if minimum update duration is set") +} + +func TestUsageService_IncreaseForTable_WithMinimumUpdateDuration(t *testing.T) { + s := createTestServer(t) + defer s.server.Close() + + apiClient, err := cqapi.NewClientWithResponses(s.server.URL) + require.NoError(t, err) + + usageClient := newClient(t, apiClient, WithBatchLimit(0), WithMinTimeBetweenFlushes(30*time.Second)) + + tables := 3 + rows := 9999 + for i := 0; i < rows; i++ { + table := "table:" + strconv.Itoa(i%tables) + err = usageClient.IncreaseForTable(table, 1) + require.NoError(t, err) + } + err = usageClient.Close() + require.NoError(t, err) + + assert.Equal(t, rows, s.sumOfUpdates(), "total should equal number of updated rows") + assert.Equal(t, rows, s.sumOfTableUpdates(), "breakdown over tables should equal number of updated rows") + assert.Equal(t, 2, s.numberOfUpdates(), "should only update first time and on close if minimum update duration is set") +} + +func TestUsageService_IncreaseForTable_CorrectByTable(t *testing.T) { + s := createTestServer(t) + defer s.server.Close() + + apiClient, err := cqapi.NewClientWithResponses(s.server.URL) + require.NoError(t, err) + + usageClient := newClient(t, apiClient, WithBatchLimit(50)) + + tables := 9 + rows := 9999 + for i := 0; i < rows; i++ { + table := "table:" + strconv.Itoa(i%tables) + err = usageClient.IncreaseForTable(table, 1) + require.NoError(t, err) + } + + err = usageClient.Close() + require.NoError(t, err) + + assert.Equal(t, rows, s.sumOfUpdates(), "total should equal number of updated rows") + assert.Equal(t, rows, s.sumOfTableUpdates(), "breakdown over tables should equal number of updated rows") + + for i := 0; i < tables; i++ { + assert.Equal(t, 1111, s.tables["table:"+strconv.Itoa(i)].Rows, "table should have correct number of rows") + } +} + +func TestUsageService_Increase_ErrorOnMixingMethods(t *testing.T) { + s := createTestServer(t) + defer s.server.Close() + + apiClient, err := cqapi.NewClientWithResponses(s.server.URL) + require.NoError(t, err) + + usageClient := newClient(t, apiClient, WithBatchLimit(50)) + + assert.Equal(t, usageClient.usageIncreaseMethod, UsageIncreaseMethodUnset, "usage increase method should not be set") + + err = usageClient.Increase(1) + require.NoError(t, err) + + assert.Equal(t, usageClient.usageIncreaseMethod, UsageIncreaseMethodTotal, "usage increase method should be total") + + err = usageClient.IncreaseForTable("test_table", 1) + require.ErrorContains(t, err, "mixing usage increase methods is not allowed") + + err = usageClient.Close() + require.NoError(t, err) + + assert.Equal(t, 1, s.sumOfUpdates(), "total should equal number of updated rows") + assert.Equal(t, 0, s.sumOfTableUpdates(), "breakdown over tables should equal number of updated rows") +} + +func TestUsageService_IncreaseForTable_ErrorOnMixingMethods(t *testing.T) { + s := createTestServer(t) + defer s.server.Close() + + apiClient, err := cqapi.NewClientWithResponses(s.server.URL) + require.NoError(t, err) + + usageClient := newClient(t, apiClient, WithBatchLimit(50)) + + assert.Equal(t, usageClient.usageIncreaseMethod, UsageIncreaseMethodUnset, "usage increase method should not be set") + + err = usageClient.IncreaseForTable("test_table", 1) + require.NoError(t, err) + + assert.Equal(t, usageClient.usageIncreaseMethod, UsageIncreaseMethodBreakdown, "usage increase method should be breakdown") + + err = usageClient.Increase(1) + require.ErrorContains(t, err, "mixing usage increase methods is not allowed") + + err = usageClient.Close() + require.NoError(t, err) + + assert.Equal(t, 1, s.sumOfUpdates(), "total should equal number of updated rows") + assert.Equal(t, 1, s.sumOfTableUpdates(), "breakdown over tables should equal number of updated rows") +} + +func TestUsageService_Increase_Header(t *testing.T) { + s := createTestServer(t) + s.headers[BatchLimitHeader] = "1000" + s.headers[MinimumUpdateIntervalHeader] = "60" + s.headers[MaximumUpdateIntervalHeader] = "120" + defer s.server.Close() + + apiClient, err := cqapi.NewClientWithResponses(s.server.URL) + require.NoError(t, err) + + usageClient := newClient(t, apiClient, WithBatchLimit(50)) + + // Initial configuration + assert.Equal(t, uint32(50), usageClient.batchLimit) + assert.Equal(t, 10*time.Second, usageClient.minTimeBetweenFlushes) + assert.Equal(t, 30*time.Second, usageClient.maxTimeBetweenFlushes) + + // Generate some usage + err = usageClient.Increase(100) + require.NoError(t, err) + err = usageClient.Close() + require.NoError(t, err) + + // Check the resulting configuration + assert.Equal(t, 1, s.numberOfUpdates()) + assert.Equal(t, uint32(1000), usageClient.batchLimit) + assert.Equal(t, 60*time.Second, usageClient.minTimeBetweenFlushes) + assert.Equal(t, 120*time.Second, usageClient.maxTimeBetweenFlushes) +} + +func TestUsageService_NoUpdates(t *testing.T) { + s := createTestServer(t) + defer s.server.Close() + + apiClient, err := cqapi.NewClientWithResponses(s.server.URL) + require.NoError(t, err) + + usageClient := newClient(t, apiClient, WithBatchLimit(0)) + + err = usageClient.Close() + require.NoError(t, err) + + assert.Equal(t, 0, s.numberOfUpdates(), "total number of updates should be zero") +} + +func TestUsageService_UpdatesWithZeroRows(t *testing.T) { + s := createTestServer(t) + defer s.server.Close() + + apiClient, err := cqapi.NewClientWithResponses(s.server.URL) + require.NoError(t, err) + + usageClient := newClient(t, apiClient, WithBatchLimit(0)) + + err = usageClient.Increase(0) + require.Error(t, err, "should not be able to update with zero rows") + + err = usageClient.Close() + require.NoError(t, err) + + assert.Equal(t, 0, s.numberOfUpdates(), "total number of updates should be zero") +} + +func TestUsageService_ShouldNotUpdateClosedService(t *testing.T) { + s := createTestServer(t) + defer s.server.Close() + + apiClient, err := cqapi.NewClientWithResponses(s.server.URL) + require.NoError(t, err) + + usageClient := newClient(t, apiClient, WithBatchLimit(0)) + + // Close the service first + err = usageClient.Close() + require.NoError(t, err) + + err = usageClient.Increase(10) + require.Error(t, err, "should not be able to update closed service") + + assert.Equal(t, 0, s.numberOfUpdates(), "total number of updates should be zero") +} + +func TestUsageService_CalculateRetryDuration_Exp(t *testing.T) { + tests := []struct { + name string + statusCode int + headers http.Header + retry int + expectedSeconds int + ops func(client *BatchUpdater) + }{ + { + name: "first retry", + statusCode: http.StatusServiceUnavailable, + headers: http.Header{}, + retry: 0, + expectedSeconds: 1, + }, + { + name: "second retry", + statusCode: http.StatusServiceUnavailable, + headers: http.Header{}, + retry: 1, + expectedSeconds: 2, + }, + { + name: "third retry", + statusCode: http.StatusServiceUnavailable, + headers: http.Header{}, + retry: 2, + expectedSeconds: 4, + }, + { + name: "fourth retry", + statusCode: http.StatusServiceUnavailable, + headers: http.Header{}, + retry: 3, + expectedSeconds: 8, + }, + { + name: "should max out at max wait time", + statusCode: http.StatusServiceUnavailable, + headers: http.Header{}, + retry: 10, + expectedSeconds: 30, + ops: func(client *BatchUpdater) { + client.maxWaitTime = 30 * time.Second + }, + }, + } + + for _, tt := range tests { + usageClient := newClient(t, nil) + if tt.ops != nil { + tt.ops(usageClient) + } + t.Run(tt.name, func(t *testing.T) { + retryDuration, err := usageClient.calculateRetryDuration(tt.statusCode, tt.headers, time.Now(), tt.retry) + require.NoError(t, err) + + assert.InDeltaf(t, tt.expectedSeconds, retryDuration.Seconds(), 1, "retry duration should be %d seconds", tt.expectedSeconds) + }) + } +} + +func TestUsageService_CalculateRetryDuration_ServerBackPressure(t *testing.T) { + tests := []struct { + name string + statusCode int + headers http.Header + retry int + expectedSeconds int + ops func(client *BatchUpdater) + wantErr error + }{ + { + name: "should use exponential backoff on 503 and no header", + statusCode: http.StatusServiceUnavailable, + headers: http.Header{}, + retry: 0, + expectedSeconds: 1, + }, + { + name: "should use exponential backoff on 429 if no retry-after header", + statusCode: http.StatusTooManyRequests, + headers: http.Header{}, + retry: 1, + expectedSeconds: 2, + }, + { + name: "should use retry-after header if present on 429", + statusCode: http.StatusTooManyRequests, + headers: http.Header{"Retry-After": []string{"5"}}, + retry: 0, + expectedSeconds: 5, + }, + } + + for _, tt := range tests { + usageClient := newClient(t, nil) + if tt.ops != nil { + tt.ops(usageClient) + } + t.Run(tt.name, func(t *testing.T) { + retryDuration, err := usageClient.calculateRetryDuration(tt.statusCode, tt.headers, time.Now(), tt.retry) + if tt.wantErr == nil { + require.NoError(t, err) + } else { + assert.Contains(t, err.Error(), tt.wantErr.Error()) + } + + assert.InDeltaf(t, tt.expectedSeconds, retryDuration.Seconds(), 1, "retry duration should be %d seconds", tt.expectedSeconds) + }) + } +} + +func newClient(t *testing.T, apiClient *cqapi.ClientWithResponses, ops ...UsageClientOptions) *BatchUpdater { + client, err := NewUsageClient( + plugin.Meta{ + Team: "plugin-team", + Kind: cqapi.PluginKindSource, + Name: "vault", + }, + append(ops, withTeamName("team-name"), WithAPIClient(apiClient), withTokenClient(newMockTokenClient(auth.BearerToken)))...) + require.NoError(t, err) + + return client.(*BatchUpdater) +} + +func createTestServerWithRemainingRows(t *testing.T, remainingRows int) *testStage { + stage := testStage{ + remainingRows: remainingRows, + headers: make(map[string]string), + update: make([]int, 0), + tables: make(map[string]struct { + Name string + Rows int + }), + } + + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == "GET" { + w.Header().Set("Content-Type", "application/json") + if _, err := fmt.Fprintf(w, `{"remaining_rows": %d}`, stage.remainingRows); err != nil { + t.Fatal(err) + } + w.WriteHeader(http.StatusOK) + return + } + if r.Method == "POST" { + dec := json.NewDecoder(r.Body) + var req cqapi.IncreaseTeamPluginUsageJSONRequestBody + err := dec.Decode(&req) + require.NoError(t, err) + + stage.mu.Lock() + defer stage.mu.Unlock() + stage.update = append(stage.update, req.Rows) + + if req.Tables != nil { + for _, table := range *req.Tables { + tbl, ok := stage.tables[table.Name] + + if !ok { + stage.tables[table.Name] = struct { + Name string + Rows int + }{Name: table.Name, Rows: table.Rows} + continue + } + + tbl.Rows += table.Rows + stage.tables[table.Name] = tbl + } + } + + // Set response headers + for k, v := range stage.headers { + w.Header().Set(k, v) + } + + w.WriteHeader(http.StatusOK) + return + } + }) + + stage.server = httptest.NewServer(handler) + + return &stage +} + +func createTestServer(t *testing.T) *testStage { + return createTestServerWithRemainingRows(t, 0) +} + +type testStage struct { + server *httptest.Server + + headers map[string]string + + remainingRows int + update []int + tables map[string]struct { + Name string + Rows int + } + mu sync.RWMutex +} + +func (s *testStage) numberOfUpdates() int { + s.mu.RLock() + defer s.mu.RUnlock() + return len(s.update) +} + +func (s *testStage) sumOfUpdates() int { + s.mu.RLock() + defer s.mu.RUnlock() + sum := 0 + for _, val := range s.update { + sum += val + } + return sum +} + +func (s *testStage) sumOfTableUpdates() int { + s.mu.RLock() + defer s.mu.RUnlock() + sum := 0 + for _, val := range s.tables { + sum += val.Rows + } + return sum +} + +func (s *testStage) minExcludingClose() int { + s.mu.RLock() + defer s.mu.RUnlock() + m := math.MaxInt + for i := 0; i < len(s.update); i++ { + if s.update[i] < m { + m = s.update[i] + } + } + return m +} + +func Test_UsageClientInit_FromManagedSyncsAPIKeys(t *testing.T) { + type testCase struct { + name string + envs map[string]string + err string + } + testCases := []testCase{ + { + name: "sync run API key with team name", + envs: map[string]string{ + auth.EnvVarCloudQueryAPIKey: "cqsr_api_key", + "_CQ_TEAM_NAME": "cqrn_team_name", + }, + }, + { + name: "sync run API key no team name", + envs: map[string]string{ + auth.EnvVarCloudQueryAPIKey: "cqsr_api_key", + }, + err: "failed to get team name: _CQ_TEAM_NAME environment variable not set", + }, + { + name: "sync test connection API key with team name", + envs: map[string]string{ + auth.EnvVarCloudQueryAPIKey: "cqstc_api_key", + "_CQ_TEAM_NAME": "cqstc_team_name", + }, + }, + { + name: "sync test connection API key no team name", + envs: map[string]string{ + auth.EnvVarCloudQueryAPIKey: "cqstc_api_key", + }, + err: "failed to get team name: _CQ_TEAM_NAME environment variable not set", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + for k, v := range tc.envs { + t.Setenv(k, v) + } + + _, err := NewUsageClient( + plugin.Meta{ + Team: "plugin-team", + Kind: cqapi.PluginKindSource, + Name: "test", + }, + ) + if tc.err != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.err) + } else { + require.NoError(t, err) + } + }) + } +} + +func Test_UsageClientInit_UnknownTokenType(t *testing.T) { + type testCase struct { + name string + envs map[string]string + err string + } + testCases := []testCase{ + { + name: "unknown API key with team name", + envs: map[string]string{ + "_CQ_TEAM_NAME": "team_name", + }, + }, + { + name: "unknown API key no team name", + envs: map[string]string{}, + err: "unsupported token type:", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + for k, v := range tc.envs { + t.Setenv(k, v) + } + + _, err := NewUsageClient( + plugin.Meta{ + Team: "plugin-team", + Kind: cqapi.PluginKindSource, + Name: "test", + }, + withTokenClient(newMockTokenClient(math.MaxInt)), + ) + if tc.err != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.err) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/plugin-sdk/scalar/LICENSE b/plugin-sdk/scalar/LICENSE new file mode 100644 index 00000000..530597e6 --- /dev/null +++ b/plugin-sdk/scalar/LICENSE @@ -0,0 +1,23 @@ +Copyright (c) 2013-2021 Jack Christensen +Copyright (c) 2022 CloudQuery inc. + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/plugin-sdk/scalar/README.md b/plugin-sdk/scalar/README.md new file mode 100644 index 00000000..e4090ea3 --- /dev/null +++ b/plugin-sdk/scalar/README.md @@ -0,0 +1,4 @@ +# CloudQuery Type System + +This directory is heavily based on [jackc/pgtype](https://github.com/jackc/pgtype) and modified per CQ needs +and thus fall under the original [MIT license and copyright](./LICENSE). diff --git a/plugin-sdk/scalar/binary.go b/plugin-sdk/scalar/binary.go new file mode 100644 index 00000000..e6519e05 --- /dev/null +++ b/plugin-sdk/scalar/binary.go @@ -0,0 +1,96 @@ +package scalar + +import ( + "bytes" + + "github.com/apache/arrow/go/v16/arrow" +) + +type Binary struct { + Valid bool + Value []byte +} + +func (s *Binary) IsValid() bool { + return s.Valid +} + +func (s *Binary) Equal(rhs Scalar) bool { + if rhs == nil { + return false + } + r, ok := rhs.(*Binary) + if !ok { + return false + } + return s.Valid == r.Valid && bytes.Equal(s.Value, r.Value) +} + +func (s *Binary) String() string { + if !s.Valid { + return nullValueStr + } + return string(s.Value) +} + +func (s *Binary) Get() any { + return s.Value +} + +func (s *Binary) Set(val any) error { + if val == nil { + s.Valid = false + return nil + } + + if sc, ok := val.(Scalar); ok { + if !sc.IsValid() { + s.Valid = false + return nil + } + return s.Set(sc.Get()) + } + + switch value := val.(type) { + case *[]byte: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case []byte: + if value == nil { + s.Valid = false + return nil + } + s.Value = value + case string: + s.Value = []byte(value) + case *string: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + default: + if originalSrc, ok := underlyingBytesType(value); ok { + return s.Set(originalSrc) + } + return &ValidationError{Type: arrow.BinaryTypes.Binary, Msg: noConversion, Value: value} + } + + s.Valid = true + return nil +} + +func (*Binary) DataType() arrow.DataType { + return arrow.BinaryTypes.Binary +} + +type LargeBinary struct { + Binary +} + +func (*LargeBinary) DataType() arrow.DataType { + return arrow.BinaryTypes.LargeBinary +} diff --git a/plugin-sdk/scalar/binary_test.go b/plugin-sdk/scalar/binary_test.go new file mode 100644 index 00000000..d8163843 --- /dev/null +++ b/plugin-sdk/scalar/binary_test.go @@ -0,0 +1,33 @@ +package scalar + +import "testing" + +func TestBinarySet(t *testing.T) { + var nilPointerByteArray *[]byte + var nilPointerString *string + + successfulTests := []struct { + source any + result Binary + }{ + {source: []byte{1, 2, 3}, result: Binary{Value: []byte{1, 2, 3}, Valid: true}}, + {source: []byte{}, result: Binary{Value: []byte{}, Valid: true}}, + {source: []byte(nil), result: Binary{}}, + {source: _byteSlice{1, 2, 3}, result: Binary{Value: []byte{1, 2, 3}, Valid: true}}, + {source: _byteSlice(nil), result: Binary{}}, + {source: nilPointerByteArray, result: Binary{}}, + {source: nilPointerString, result: Binary{}}, + } + + for i, tt := range successfulTests { + var r Binary + err := r.Set(tt.source) + if err != nil { + t.Errorf("%d: %v", i, err) + } + + if !r.Equal(&tt.result) { + t.Errorf("%d: %v != %v", i, r, tt.result) + } + } +} diff --git a/plugin-sdk/scalar/bool.go b/plugin-sdk/scalar/bool.go new file mode 100644 index 00000000..862d96d5 --- /dev/null +++ b/plugin-sdk/scalar/bool.go @@ -0,0 +1,87 @@ +package scalar + +import ( + "strconv" + + "github.com/apache/arrow/go/v16/arrow" +) + +type Bool struct { + Valid bool + Value bool +} + +func (s *Bool) IsValid() bool { + return s.Valid +} + +func (*Bool) DataType() arrow.DataType { + return arrow.FixedWidthTypes.Boolean +} + +func (s *Bool) Equal(rhs Scalar) bool { + if rhs == nil { + return false + } + r, ok := rhs.(*Bool) + if !ok { + return false + } + return s.Value == r.Value && s.Valid == r.Valid +} + +func (s *Bool) String() string { + if !s.Valid { + return nullValueStr + } + return strconv.FormatBool(s.Value) +} + +func (s *Bool) Get() any { + return s.Value +} + +func (s *Bool) Set(val any) error { + if val == nil { + s.Valid = false + return nil + } + + if sc, ok := val.(Scalar); ok { + if !sc.IsValid() { + s.Valid = false + return nil + } + return s.Set(sc.Get()) + } + + switch value := val.(type) { + case bool: + s.Value = value + case string: + bb, err := strconv.ParseBool(value) + if err != nil { + return &ValidationError{Type: arrow.FixedWidthTypes.Boolean, Msg: "failed to ParseBool", Value: value, Err: err} + } + s.Value = bb + case *bool: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *string: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + default: + if originalSrc, ok := underlyingBoolType(value); ok { + return s.Set(originalSrc) + } + return &ValidationError{Type: arrow.FixedWidthTypes.Boolean, Msg: noConversion, Value: val} + } + s.Valid = true + return nil +} diff --git a/plugin-sdk/scalar/bool_test.go b/plugin-sdk/scalar/bool_test.go new file mode 100644 index 00000000..f8b7660d --- /dev/null +++ b/plugin-sdk/scalar/bool_test.go @@ -0,0 +1,39 @@ +package scalar + +import ( + "testing" +) + +func TestBoolSet(t *testing.T) { + var nilPointerBool *bool + var nilPointerString *string + + successfulTests := []struct { + source any + result Bool + }{ + {source: true, result: Bool{Value: true, Valid: true}}, + {source: false, result: Bool{Value: false, Valid: true}}, + {source: "true", result: Bool{Value: true, Valid: true}}, + {source: "false", result: Bool{Value: false, Valid: true}}, + {source: "t", result: Bool{Value: true, Valid: true}}, + {source: "f", result: Bool{Value: false, Valid: true}}, + {source: _bool(true), result: Bool{Value: true, Valid: true}}, + {source: _bool(false), result: Bool{Value: false, Valid: true}}, + {source: &Bool{Value: true, Valid: true}, result: Bool{Value: true, Valid: true}}, + {source: nil, result: Bool{}}, + {source: nilPointerBool, result: Bool{}}, + {source: nilPointerString, result: Bool{}}, + } + + for i, tt := range successfulTests { + var r Bool + err := r.Set(tt.source) + if err != nil { + t.Errorf("%d: %v", i, err) + } + if !r.Equal(&tt.result) { + t.Errorf("%d: %v != %v", i, r, tt.result) + } + } +} diff --git a/plugin-sdk/scalar/convert.go b/plugin-sdk/scalar/convert.go new file mode 100644 index 00000000..c40f0acd --- /dev/null +++ b/plugin-sdk/scalar/convert.go @@ -0,0 +1,185 @@ +package scalar + +import ( + "reflect" + "time" +) + +// underlyingNumberType gets the underlying type that can be converted to Int2, Int4, Int8, Float4, or Float8 +func underlyingNumberType(val any) (any, bool) { + refVal := reflect.ValueOf(val) + + switch refVal.Kind() { + case reflect.Ptr: + if refVal.IsNil() { + return nil, false + } + convVal := refVal.Elem().Interface() + return convVal, true + case reflect.Int: + convVal := int(refVal.Int()) + a := reflect.TypeOf(convVal) + b := refVal.Type() + return convVal, a != b + case reflect.Int8: + convVal := int8(refVal.Int()) + a := reflect.TypeOf(convVal) + b := refVal.Type() + return convVal, a != b + case reflect.Int16: + convVal := int16(refVal.Int()) + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Int32: + convVal := int32(refVal.Int()) + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Int64: + convVal := refVal.Int() + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Uint: + convVal := uint(refVal.Uint()) + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Uint8: + convVal := uint8(refVal.Uint()) + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Uint16: + convVal := uint16(refVal.Uint()) + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Uint32: + convVal := uint32(refVal.Uint()) + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Uint64: + convVal := refVal.Uint() + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Float32: + convVal := float32(refVal.Float()) + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Float64: + convVal := refVal.Float() + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.String: + convVal := refVal.String() + return convVal, reflect.TypeOf(convVal) != refVal.Type() + } + + return nil, false +} + +// underlyingStringType gets the underlying type that can be converted to String +func underlyingStringType(val any) (any, bool) { + refVal := reflect.ValueOf(val) + + switch refVal.Kind() { + case reflect.Ptr: + if refVal.IsNil() { + return nil, false + } + convVal := refVal.Elem().Interface() + return convVal, true + case reflect.String: + convVal := refVal.String() + return convVal, reflect.TypeOf(convVal) != refVal.Type() + } + + return nil, false +} + +// underlyingBoolType gets the underlying type that can be converted to Bool +func underlyingBoolType(val any) (any, bool) { + refVal := reflect.ValueOf(val) + + switch refVal.Kind() { + case reflect.Ptr: + if refVal.IsNil() { + return nil, false + } + convVal := refVal.Elem().Interface() + return convVal, true + case reflect.Bool: + convVal := refVal.Bool() + return convVal, reflect.TypeOf(convVal) != refVal.Type() + } + + return nil, false +} + +// underlyingBytesType gets the underlying type that can be converted to []byte +func underlyingBytesType(val any) (any, bool) { + refVal := reflect.ValueOf(val) + + switch refVal.Kind() { + case reflect.Ptr: + if refVal.IsNil() { + return nil, false + } + convVal := refVal.Elem().Interface() + return convVal, true + case reflect.Slice: + if refVal.Type().Elem().Kind() == reflect.Uint8 { + convVal := refVal.Bytes() + return convVal, reflect.TypeOf(convVal) != refVal.Type() + } + } + + return nil, false +} + +// underlyingTimeType gets the underlying type that can be converted to time.Time +func underlyingTimeType(val any) (any, bool) { + refVal := reflect.ValueOf(val) + + // nolint:gocritic,revive + switch refVal.Kind() { + case reflect.Ptr: + if refVal.IsNil() { + return nil, false + } + convVal := refVal.Elem().Interface() + return convVal, true + } + + timeType := reflect.TypeOf(time.Time{}) + if refVal.Type().ConvertibleTo(timeType) { + return refVal.Convert(timeType).Interface(), true + } + + return nil, false +} + +// underlyingUUIDType gets the underlying type that can be converted to [16]byte +func underlyingUUIDType(val any) (any, bool) { + refVal := reflect.ValueOf(val) + + //nolint:revive,gocritic + switch refVal.Kind() { + case reflect.Ptr: + if refVal.IsNil() { + return nil, false + } + convVal := refVal.Elem().Interface() + return convVal, true + } + + uuidType := reflect.TypeOf([16]byte{}) + if refVal.Type().ConvertibleTo(uuidType) { + return refVal.Convert(uuidType).Interface(), true + } + + return nil, false +} + +// underlyingPtrType dereferences a pointer +func underlyingPtrType(val any) (any, bool) { + refVal := reflect.ValueOf(val) + + //nolint:gocritic,revive + switch refVal.Kind() { + case reflect.Ptr: + if refVal.IsNil() { + return nil, false + } + convVal := refVal.Elem().Interface() + return convVal, true + } + + return nil, false +} diff --git a/plugin-sdk/scalar/date32.go b/plugin-sdk/scalar/date32.go new file mode 100644 index 00000000..8df72e03 --- /dev/null +++ b/plugin-sdk/scalar/date32.go @@ -0,0 +1,107 @@ +package scalar + +import ( + "encoding" + "fmt" + "time" + + "github.com/apache/arrow/go/v16/arrow" +) + +type Date32 struct { + Valid bool + Value arrow.Date32 +} + +func (s *Date32) IsValid() bool { + return s.Valid +} + +func (*Date32) DataType() arrow.DataType { + return arrow.FixedWidthTypes.Date32 +} + +func (s *Date32) Equal(rhs Scalar) bool { + if rhs == nil { + return false + } + r, ok := rhs.(*Date32) + if !ok { + return false + } + return s.Valid == r.Valid && s.Value == r.Value +} + +func (s *Date32) String() string { + if !s.Valid { + return nullValueStr + } + return s.Value.FormattedString() +} + +func (s *Date32) Get() any { + return s.Value +} + +func (s *Date32) Set(val any) error { + if val == nil { + s.Valid = false + return nil + } + + if sc, ok := val.(Scalar); ok { + if !sc.IsValid() { + s.Valid = false + return nil + } + return s.Set(sc.Get()) + } + + switch value := val.(type) { + case arrow.Date32: + s.Value = value + case time.Time: + return s.Set(arrow.Date32FromTime(value)) + case *time.Time: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case string: + if value == "" { + s.Valid = false + return nil + } + + p, err := time.Parse("2006-01-02", value) + if err != nil { + return &ValidationError{Type: s.DataType(), Msg: "cannot parse date", Value: value, Err: err} + } + return s.Set(p) + case *string: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + default: + if originalSrc, ok := underlyingTimeType(val); ok { + return s.Set(originalSrc) + } + if value, ok := value.(encoding.TextMarshaler); ok { + text, err := value.MarshalText() + if err == nil { + return s.Set(string(text)) + } + // fall through to String() method + } + if value, ok := value.(fmt.Stringer); ok { + str := value.String() + return s.Set(str) + } + return &ValidationError{Type: s.DataType(), Msg: noConversion, Value: value} + } + s.Valid = true + return nil +} diff --git a/plugin-sdk/scalar/date32_test.go b/plugin-sdk/scalar/date32_test.go new file mode 100644 index 00000000..774c6bb3 --- /dev/null +++ b/plugin-sdk/scalar/date32_test.go @@ -0,0 +1,33 @@ +package scalar + +import ( + "testing" + "time" + + "github.com/apache/arrow/go/v16/arrow" +) + +func TestDate32Set(t *testing.T) { + successfulTests := []struct { + source any + result Date32 + }{ + {source: time.Date(1900, 1, 1, 0, 0, 0, 0, time.UTC), result: Date32{Value: arrow.Date32FromTime(time.Date(1900, 1, 1, 0, 0, 0, 0, time.UTC)), Valid: true}}, + {source: time.Date(1999, 12, 31, 12, 59, 59, 0, time.UTC), result: Date32{Value: arrow.Date32FromTime(time.Date(1999, 12, 31, 0, 0, 0, 0, time.UTC)), Valid: true}}, + {source: "2150-10-15", result: Date32{Value: arrow.Date32FromTime(time.Date(2150, 10, 15, 0, 0, 0, 0, time.UTC)), Valid: true}}, + {source: "", result: Date32{}}, + } + + for i, tt := range successfulTests { + var r Date32 + err := r.Set(tt.source) + if err != nil { + t.Errorf("%d: %v", i, err) + continue + } + + if !r.Equal(&tt.result) { + t.Errorf("%d: %v != %v", i, r, tt.result) + } + } +} diff --git a/plugin-sdk/scalar/date64.go b/plugin-sdk/scalar/date64.go new file mode 100644 index 00000000..d32c526b --- /dev/null +++ b/plugin-sdk/scalar/date64.go @@ -0,0 +1,107 @@ +package scalar + +import ( + "encoding" + "fmt" + "time" + + "github.com/apache/arrow/go/v16/arrow" +) + +type Date64 struct { + Valid bool + Value arrow.Date64 +} + +func (s *Date64) IsValid() bool { + return s.Valid +} + +func (*Date64) DataType() arrow.DataType { + return arrow.FixedWidthTypes.Date64 +} + +func (s *Date64) Equal(rhs Scalar) bool { + if rhs == nil { + return false + } + r, ok := rhs.(*Date64) + if !ok { + return false + } + return s.Valid == r.Valid && s.Value == r.Value +} + +func (s *Date64) String() string { + if !s.Valid { + return nullValueStr + } + return s.Value.FormattedString() +} + +func (s *Date64) Get() any { + return s.Value +} + +func (s *Date64) Set(val any) error { + if val == nil { + s.Valid = false + return nil + } + + if sc, ok := val.(Scalar); ok { + if !sc.IsValid() { + s.Valid = false + return nil + } + return s.Set(sc.Get()) + } + + switch value := val.(type) { + case arrow.Date64: + s.Value = value + case time.Time: + return s.Set(arrow.Date64FromTime(value)) + case *time.Time: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case string: + if value == "" { + s.Valid = false + return nil + } + + p, err := time.Parse("2006-01-02", value) + if err != nil { + return &ValidationError{Type: s.DataType(), Msg: "cannot parse date", Value: value, Err: err} + } + return s.Set(p) + case *string: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + default: + if originalSrc, ok := underlyingTimeType(val); ok { + return s.Set(originalSrc) + } + if value, ok := value.(encoding.TextMarshaler); ok { + text, err := value.MarshalText() + if err == nil { + return s.Set(string(text)) + } + // fall through to String() method + } + if value, ok := value.(fmt.Stringer); ok { + str := value.String() + return s.Set(str) + } + return &ValidationError{Type: s.DataType(), Msg: noConversion, Value: value} + } + s.Valid = true + return nil +} diff --git a/plugin-sdk/scalar/date64_test.go b/plugin-sdk/scalar/date64_test.go new file mode 100644 index 00000000..b40145c3 --- /dev/null +++ b/plugin-sdk/scalar/date64_test.go @@ -0,0 +1,31 @@ +package scalar + +import ( + "testing" + "time" +) + +func TestDate64Set(t *testing.T) { + successfulTests := []struct { + source any + result Date64 + }{ + {source: time.Date(1900, 1, 1, 0, 0, 1, 0, time.UTC), result: Date64{Value: -2208988800000, Valid: true}}, + {source: time.Date(1999, 12, 31, 12, 59, 59, 0, time.UTC), result: Date64{Value: 946598400000, Valid: true}}, + {source: "2150-10-15", result: Date64{Value: 5705078400000, Valid: true}}, + {source: "", result: Date64{}}, + } + + for i, tt := range successfulTests { + var r Date64 + err := r.Set(tt.source) + if err != nil { + t.Errorf("%d: %v", i, err) + continue + } + + if !r.Equal(&tt.result) { + t.Errorf("%d: %v != %v", i, r, tt.result) + } + } +} diff --git a/plugin-sdk/scalar/decimal.go b/plugin-sdk/scalar/decimal.go new file mode 100644 index 00000000..487675bd --- /dev/null +++ b/plugin-sdk/scalar/decimal.go @@ -0,0 +1,337 @@ +package scalar + +import ( + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" +) + +type Decimal256 struct { + Valid bool + Value decimal256.Num + Type *arrow.Decimal256Type // Stores precision and scale +} + +func (s *Decimal256) IsValid() bool { + return s.Valid +} + +func (s *Decimal256) DataType() arrow.DataType { + return s.Type +} + +func (s *Decimal256) Equal(rhs Scalar) bool { + if rhs == nil { + return false + } + r, ok := rhs.(*Decimal256) + if !ok { + return false + } + return s.Valid == r.Valid && s.Value == r.Value +} + +func (s *Decimal256) String() string { + if !s.Valid { + return nullValueStr + } + return s.Value.ToString(s.Type.Scale) +} + +func (s *Decimal256) Get() any { + return s.Value +} + +func (s *Decimal256) Set(val any) error { + if val == nil { + s.Valid = false + return nil + } + + if sc, ok := val.(Scalar); ok { + if !sc.IsValid() { + s.Valid = false + return nil + } + return s.Set(sc.Get()) + } + + switch value := val.(type) { + case decimal256.Num: + s.Value = value + case decimal128.Num: + s.Value = decimal256.FromDecimal128(value) + case *decimal256.Num: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *decimal128.Num: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case int: + s.Value = decimal256.FromI64(int64(value)) + case int8: + s.Value = decimal256.FromI64(int64(value)) + case int16: + s.Value = decimal256.FromI64(int64(value)) + case int32: + s.Value = decimal256.FromI64(int64(value)) + case int64: + s.Value = decimal256.FromI64(value) + case uint: + s.Value = decimal256.FromU64(uint64(value)) + case uint8: + s.Value = decimal256.FromU64(uint64(value)) + case uint16: + s.Value = decimal256.FromU64(uint64(value)) + case uint32: + s.Value = decimal256.FromU64(uint64(value)) + case uint64: + s.Value = decimal256.FromU64(value) + case string: + v, err := decimal256.FromString(value, s.Type.Precision, s.Type.Scale) + if err != nil { + return err + } + s.Value = v + case *int: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *int8: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *int16: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *int32: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *int64: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *uint: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *uint8: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *uint16: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *uint32: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *uint64: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *string: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + default: + if originalSrc, ok := underlyingPtrType(val); ok { + return s.Set(originalSrc) + } + return &ValidationError{Type: s.DataType(), Msg: noConversion, Value: value} + } + s.Valid = true + return nil +} + +type Decimal128 struct { + Valid bool + Value decimal128.Num + Type *arrow.Decimal128Type // Stores precision and scale +} + +func (s *Decimal128) IsValid() bool { + return s.Valid +} + +func (s *Decimal128) DataType() arrow.DataType { + return s.Type +} + +func (s *Decimal128) Equal(rhs Scalar) bool { + if rhs == nil { + return false + } + r, ok := rhs.(*Decimal128) + if !ok { + return false + } + return s.Valid == r.Valid && s.Value == r.Value +} + +func (s *Decimal128) String() string { + if !s.Valid { + return nullValueStr + } + return s.Value.ToString(s.Type.Scale) +} + +func (s *Decimal128) Get() any { + return s.Value +} + +func (s *Decimal128) Set(val any) error { + if val == nil { + s.Valid = false + return nil + } + + if sc, ok := val.(Scalar); ok { + if !sc.IsValid() { + s.Valid = false + return nil + } + return s.Set(sc.Get()) + } + + switch value := val.(type) { + case decimal128.Num: + s.Value = value + case *decimal128.Num: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case int: + s.Value = decimal128.FromI64(int64(value)) + case int8: + s.Value = decimal128.FromI64(int64(value)) + case int16: + s.Value = decimal128.FromI64(int64(value)) + case int32: + s.Value = decimal128.FromI64(int64(value)) + case int64: + s.Value = decimal128.FromI64(value) + case uint: + s.Value = decimal128.FromU64(uint64(value)) + case uint8: + s.Value = decimal128.FromU64(uint64(value)) + case uint16: + s.Value = decimal128.FromU64(uint64(value)) + case uint32: + s.Value = decimal128.FromU64(uint64(value)) + case uint64: + s.Value = decimal128.FromU64(value) + case string: + v, err := decimal128.FromString(value, s.Type.Precision, s.Type.Scale) + if err != nil { + return err + } + s.Value = v + case *int: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *int8: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *int16: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *int32: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *int64: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *uint: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *uint8: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *uint16: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *uint32: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *uint64: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *string: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + default: + if originalSrc, ok := underlyingPtrType(val); ok { + return s.Set(originalSrc) + } + return &ValidationError{Type: s.DataType(), Msg: noConversion, Value: value} + } + s.Valid = true + return nil +} diff --git a/plugin-sdk/scalar/decimal_test.go b/plugin-sdk/scalar/decimal_test.go new file mode 100644 index 00000000..ac1baa2b --- /dev/null +++ b/plugin-sdk/scalar/decimal_test.go @@ -0,0 +1,126 @@ +package scalar + +import ( + "testing" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/stretchr/testify/require" +) + +// nolint:dupl +func TestDecimal128Set(t *testing.T) { + str := "100.32" + decimalType := &arrow.Decimal128Type{Precision: 5, Scale: 2} + strDecimal, _ := decimal128.FromString(str, decimalType.Precision, decimalType.Scale) + + intVal := int(1) + int8Val := int8(1) + int16Val := int16(1) + int32Val := int32(1) + int64Val := int64(1) + uintVal := uint(1) + uint8Val := uint8(1) + uint16Val := uint16(1) + uint32Val := uint32(1) + uint64Val := uint64(1) + + intValPointer := &intVal + + successfulTests := []struct { + source any + decimalType *arrow.Decimal128Type + expect Decimal128 + }{ + {source: str, expect: Decimal128{Value: strDecimal, Valid: true, Type: decimalType}, decimalType: decimalType}, + {source: &str, expect: Decimal128{Value: strDecimal, Valid: true, Type: decimalType}, decimalType: decimalType}, + {source: intVal, expect: Decimal128{Value: decimal128.FromI64(1), Valid: true}}, + {source: int8Val, expect: Decimal128{Value: decimal128.FromI64(1), Valid: true}}, + {source: int16Val, expect: Decimal128{Value: decimal128.FromI64(1), Valid: true}}, + {source: int32Val, expect: Decimal128{Value: decimal128.FromI64(1), Valid: true}}, + {source: int64Val, expect: Decimal128{Value: decimal128.FromI64(1), Valid: true}}, + {source: uintVal, expect: Decimal128{Value: decimal128.FromU64(1), Valid: true}}, + {source: uint8Val, expect: Decimal128{Value: decimal128.FromU64(1), Valid: true}}, + {source: uint16Val, expect: Decimal128{Value: decimal128.FromU64(1), Valid: true}}, + {source: uint32Val, expect: Decimal128{Value: decimal128.FromU64(1), Valid: true}}, + {source: uint64Val, expect: Decimal128{Value: decimal128.FromU64(1), Valid: true}}, + {source: &intVal, expect: Decimal128{Value: decimal128.FromI64(1), Valid: true}}, + {source: &int8Val, expect: Decimal128{Value: decimal128.FromI64(1), Valid: true}}, + {source: &int16Val, expect: Decimal128{Value: decimal128.FromI64(1), Valid: true}}, + {source: &int32Val, expect: Decimal128{Value: decimal128.FromI64(1), Valid: true}}, + {source: &int64Val, expect: Decimal128{Value: decimal128.FromI64(1), Valid: true}}, + {source: &uintVal, expect: Decimal128{Value: decimal128.FromU64(1), Valid: true}}, + {source: &uint8Val, expect: Decimal128{Value: decimal128.FromU64(1), Valid: true}}, + {source: &uint16Val, expect: Decimal128{Value: decimal128.FromU64(1), Valid: true}}, + {source: &uint32Val, expect: Decimal128{Value: decimal128.FromU64(1), Valid: true}}, + {source: &uint64Val, expect: Decimal128{Value: decimal128.FromU64(1), Valid: true}}, + {source: &intValPointer, expect: Decimal128{Value: decimal128.FromI64(1), Valid: true}}, + } + + for i, tt := range successfulTests { + r := Decimal128{} + r.Type = tt.decimalType + err := r.Set(tt.source) + require.NoError(t, err, "No error expected for test %d", i) + require.Equal(t, tt.expect, r, "Unexpected result for test %d", i) + } +} + +// nolint:dupl +func TestDecimal256Set(t *testing.T) { + str := "100.32" + decimalType := &arrow.Decimal256Type{Precision: 5, Scale: 2} + strDecimal, _ := decimal256.FromString(str, decimalType.Precision, decimalType.Scale) + + intVal := int(1) + int8Val := int8(1) + int16Val := int16(1) + int32Val := int32(1) + int64Val := int64(1) + uintVal := uint(1) + uint8Val := uint8(1) + uint16Val := uint16(1) + uint32Val := uint32(1) + uint64Val := uint64(1) + + intValPointer := &intVal + + successfulTests := []struct { + source any + decimalType *arrow.Decimal256Type + expect Decimal256 + }{ + {source: str, expect: Decimal256{Value: strDecimal, Valid: true, Type: decimalType}, decimalType: decimalType}, + {source: &str, expect: Decimal256{Value: strDecimal, Valid: true, Type: decimalType}, decimalType: decimalType}, + {source: intVal, expect: Decimal256{Value: decimal256.FromI64(1), Valid: true}}, + {source: int8Val, expect: Decimal256{Value: decimal256.FromI64(1), Valid: true}}, + {source: int16Val, expect: Decimal256{Value: decimal256.FromI64(1), Valid: true}}, + {source: int32Val, expect: Decimal256{Value: decimal256.FromI64(1), Valid: true}}, + {source: int64Val, expect: Decimal256{Value: decimal256.FromI64(1), Valid: true}}, + {source: uintVal, expect: Decimal256{Value: decimal256.FromU64(1), Valid: true}}, + {source: uint8Val, expect: Decimal256{Value: decimal256.FromU64(1), Valid: true}}, + {source: uint16Val, expect: Decimal256{Value: decimal256.FromU64(1), Valid: true}}, + {source: uint32Val, expect: Decimal256{Value: decimal256.FromU64(1), Valid: true}}, + {source: uint64Val, expect: Decimal256{Value: decimal256.FromU64(1), Valid: true}}, + {source: &intVal, expect: Decimal256{Value: decimal256.FromI64(1), Valid: true}}, + {source: &int8Val, expect: Decimal256{Value: decimal256.FromI64(1), Valid: true}}, + {source: &int16Val, expect: Decimal256{Value: decimal256.FromI64(1), Valid: true}}, + {source: &int32Val, expect: Decimal256{Value: decimal256.FromI64(1), Valid: true}}, + {source: &int64Val, expect: Decimal256{Value: decimal256.FromI64(1), Valid: true}}, + {source: &uintVal, expect: Decimal256{Value: decimal256.FromU64(1), Valid: true}}, + {source: &uint8Val, expect: Decimal256{Value: decimal256.FromU64(1), Valid: true}}, + {source: &uint16Val, expect: Decimal256{Value: decimal256.FromU64(1), Valid: true}}, + {source: &uint32Val, expect: Decimal256{Value: decimal256.FromU64(1), Valid: true}}, + {source: &uint64Val, expect: Decimal256{Value: decimal256.FromU64(1), Valid: true}}, + {source: &intValPointer, expect: Decimal256{Value: decimal256.FromI64(1), Valid: true}}, + } + + for i, tt := range successfulTests { + r := Decimal256{} + r.Type = tt.decimalType + err := r.Set(tt.source) + require.NoError(t, err, "No error expected for test %d", i) + require.Equal(t, tt.expect, r, "Unexpected result for test %d", i) + } +} diff --git a/plugin-sdk/scalar/duration.go b/plugin-sdk/scalar/duration.go new file mode 100644 index 00000000..d19d0e34 --- /dev/null +++ b/plugin-sdk/scalar/duration.go @@ -0,0 +1,62 @@ +package scalar + +import ( + "strings" + "time" + + "github.com/apache/arrow/go/v16/arrow" +) + +type Duration struct { + Int + Unit arrow.TimeUnit +} + +func (s *Duration) DataType() arrow.DataType { + return &arrow.DurationType{Unit: s.Unit} +} + +func (s *Duration) String() string { + if !s.Int.IsValid() { + return nullValueStr + } + + return s.Int.String() + s.Unit.String() +} + +func (s *Duration) Equal(rhs Scalar) bool { + if rhs == nil { + return false + } + r, ok := rhs.(*Duration) + if !ok { + return false + } + return s.Valid == r.Valid && s.Unit == r.Unit && s.Value == r.Value +} + +func (s *Duration) Set(value any) error { + if dur, ok := value.(arrow.Duration); ok { + return s.Int.Set(int64(dur)) + } + switch v := value.(type) { + case string: + stripped := strings.TrimSuffix(v, s.Unit.String()) + return s.Int.Set(stripped) + case *string: + if v == nil { + s.Valid = false + return nil + } + return s.Set(*v) + case time.Duration: + return s.Int.Set(v / s.Unit.Multiplier()) + case *time.Duration: + if v == nil { + s.Valid = false + return nil + } + return s.Set(*v) + } + return s.Int.Set(value) +} diff --git a/plugin-sdk/scalar/duration_test.go b/plugin-sdk/scalar/duration_test.go new file mode 100644 index 00000000..9b027ea0 --- /dev/null +++ b/plugin-sdk/scalar/duration_test.go @@ -0,0 +1,23 @@ +package scalar + +import ( + "testing" + "time" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/stretchr/testify/assert" +) + +func TestDurationMultiplierUp(t *testing.T) { + s := NewScalar(arrow.FixedWidthTypes.Duration_us) + dur := time.Duration(1234) * time.Millisecond + assert.NoError(t, s.Set(dur)) + assert.Equal(t, "1234000us", s.String()) +} + +func TestDurationMultiplierDown(t *testing.T) { + s := NewScalar(arrow.FixedWidthTypes.Duration_us) + dur := time.Duration(1234) * time.Nanosecond + assert.NoError(t, s.Set(dur)) + assert.Equal(t, "1us", s.String()) +} diff --git a/plugin-sdk/scalar/errors.go b/plugin-sdk/scalar/errors.go new file mode 100644 index 00000000..16546660 --- /dev/null +++ b/plugin-sdk/scalar/errors.go @@ -0,0 +1,37 @@ +package scalar + +import ( + "fmt" + + "github.com/apache/arrow/go/v16/arrow" +) + +const ( + noConversion = "no conversion available" +) + +type ValidationError struct { + Err error + Msg string + Type arrow.DataType + Value any +} + +func (e *ValidationError) Error() string { + if e.Err == nil { + return fmt.Sprintf("cannot set `%s` with value `%v`: %s", e.Type, e.Value, e.Msg) + } + return fmt.Sprintf("cannot set `%s` with value `%v`: %s (%s)", e.Type, e.Value, e.Msg, e.Err) +} + +// this prints the error without the value +func (e *ValidationError) MaskedError() string { + if e.Err == nil { + return fmt.Sprintf("cannot set `%s`: %s", e.Type, e.Msg) + } + return fmt.Sprintf("cannot set `%s`: %s (%s)", e.Type, e.Msg, e.Err) +} + +func (e *ValidationError) Unwrap() error { + return e.Err +} diff --git a/plugin-sdk/scalar/float.go b/plugin-sdk/scalar/float.go new file mode 100644 index 00000000..edfc414f --- /dev/null +++ b/plugin-sdk/scalar/float.go @@ -0,0 +1,231 @@ +package scalar + +import ( + "math" + "strconv" + + "github.com/apache/arrow/go/v16/arrow" +) + +type Float struct { + Valid bool + Value float64 + BitWidth uint8 // defaults to 64 +} + +func (s *Float) IsValid() bool { + return s.Valid +} + +func (s *Float) DataType() arrow.DataType { + switch s.getBitWidth() { + case 64: + return arrow.PrimitiveTypes.Float64 + case 32: + return arrow.PrimitiveTypes.Float32 + case 16: + return arrow.FixedWidthTypes.Float16 + default: + panic("invalid bit width") + } +} + +func (s *Float) Get() any { + return s.Value +} + +func (s *Float) Equal(rhs Scalar) bool { + if rhs == nil { + return false + } + r, ok := rhs.(*Float) + if !ok { + return false + } + return s.getBitWidth() == r.getBitWidth() && s.Valid == r.Valid && s.Value == r.Value +} + +func (s *Float) String() string { + if !s.Valid { + return nullValueStr + } + return strconv.FormatFloat(s.Value, 'f', -1, 64) +} + +func (s *Float) Set(val any) error { + if val == nil { + s.Valid = false + return nil + } + + if sc, ok := val.(Scalar); ok { + if !sc.IsValid() { + s.Valid = false + return nil + } + return s.Set(sc.Get()) + } + + const ( + minSafeValue64 = -2 << 53 + maxSafeValue64 = 2 << 53 + minSafeValue32 = -2 << 24 + maxSafeValue32 = 2 << 24 + ) + + switch value := val.(type) { + case int8: + return s.Set(int64(value)) + case int16: + return s.Set(int64(value)) + case int32: + return s.Set(int64(value)) + case int64: + switch s.getBitWidth() { + case 64: + if value > maxSafeValue64 { + return &ValidationError{Type: s.DataType(), Msg: "int64 greater than maximum safe value of 2^53", Value: value} + } + if value < minSafeValue64 { + return &ValidationError{Type: s.DataType(), Msg: "int64 less than minimum safe value of -2^53", Value: value} + } + case 32: + if value > maxSafeValue32 { + return &ValidationError{Type: s.DataType(), Msg: "int64 greater than maximum safe value of 2^24", Value: value} + } + if value < minSafeValue32 { + return &ValidationError{Type: s.DataType(), Msg: "int64 less than minimum safe value of -2^24", Value: value} + } + } + return s.Set(float64(value)) + case uint8: + return s.Set(uint64(value)) + case uint16: + return s.Set(uint64(value)) + case uint32: + return s.Set(uint64(value)) + case uint64: + switch { + case s.getBitWidth() == 64 && value > maxSafeValue64: + return &ValidationError{Type: s.DataType(), Msg: "uint64 greater than maximum safe value of 2^53", Value: value} + case s.getBitWidth() == 32 && value > maxSafeValue32: + return &ValidationError{Type: s.DataType(), Msg: "uint64 greater than maximum safe value of 2^24", Value: value} + } + return s.Set(float64(value)) + case float32: + return s.Set(float64(value)) + case float64: + if err := s.validateValue(value); err != nil { + return err + } + s.Value = value + case string: + v, err := strconv.ParseFloat(value, 64) + if err != nil { + return &ValidationError{Type: s.DataType(), Msg: "invalid string", Value: value} + } + return s.Set(v) + case *string: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *int8: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *int16: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *int32: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *int64: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *uint8: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *uint16: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *uint32: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *uint64: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *float32: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *float64: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + default: + if originalSrc, ok := underlyingNumberType(value); ok { + return s.Set(originalSrc) + } + return &ValidationError{Type: s.DataType(), Msg: noConversion, Value: value} + } + s.Valid = true + return nil +} + +func (s *Float) validateValue(value float64) error { + const maxFloat16 = 65504.0 + + switch s.getBitWidth() { + case 16: + if value > maxFloat16 { + return &ValidationError{Type: s.DataType(), Msg: "value greater than maxFloat16", Value: value} + } + if value < -maxFloat16 { + return &ValidationError{Type: s.DataType(), Msg: "value less than minFloat16", Value: value} + } + case 32: + if value > math.MaxFloat32 { + return &ValidationError{Type: s.DataType(), Msg: "value greater than MaxFloat32", Value: value} + } + if value < -math.MaxFloat32 { + return &ValidationError{Type: s.DataType(), Msg: "value less than MinFloat32", Value: value} + } + } + return nil +} + +func (s *Float) getBitWidth() uint8 { + if s.BitWidth == 0 { + return 64 // default + } + return s.BitWidth +} diff --git a/plugin-sdk/scalar/float_test.go b/plugin-sdk/scalar/float_test.go new file mode 100644 index 00000000..5ac4eed5 --- /dev/null +++ b/plugin-sdk/scalar/float_test.go @@ -0,0 +1,170 @@ +package scalar + +import ( + "math" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" +) + +var nilPointerInt8 *int8 + +var nilPointerInt16 *int16 + +var nilPointerInt32 *int32 + +var nilPointerInt64 *int64 + +var nilPointerUint8 *uint8 + +var nilPointerUint16 *uint16 + +var nilPointerUint32 *uint32 + +var nilPointerUint64 *uint64 + +var nilPointerFloat32 *float32 + +var nilPointerFloat64 *float64 + +func TestFloatAllWidthsSet(t *testing.T) { + successfulTests := []struct { + source any + expect Float + }{ + {source: float32(1), expect: Float{Value: 1, Valid: true}}, + {source: float64(1), expect: Float{Value: 1, Valid: true}}, + {source: int8(1), expect: Float{Value: 1, Valid: true}}, + {source: int16(1), expect: Float{Value: 1, Valid: true}}, + {source: int32(1), expect: Float{Value: 1, Valid: true}}, + {source: int64(1), expect: Float{Value: 1, Valid: true}}, + {source: int8(-1), expect: Float{Value: -1, Valid: true}}, + {source: int16(-1), expect: Float{Value: -1, Valid: true}}, + {source: int32(-1), expect: Float{Value: -1, Valid: true}}, + {source: int64(-1), expect: Float{Value: -1, Valid: true}}, + {source: uint8(1), expect: Float{Value: 1, Valid: true}}, + {source: uint16(1), expect: Float{Value: 1, Valid: true}}, + {source: uint32(1), expect: Float{Value: 1, Valid: true}}, + {source: uint64(1), expect: Float{Value: 1, Valid: true}}, + {source: "1", expect: Float{Value: 1, Valid: true}}, + {source: _int8(1), expect: Float{Value: 1, Valid: true}}, + {source: &Float{Value: 1, Valid: true, BitWidth: 32}, expect: Float{Value: 1, Valid: true}}, + {source: nilPointerInt8, expect: Float{Valid: false}}, + {source: nilPointerInt16, expect: Float{Valid: false}}, + {source: nilPointerInt32, expect: Float{Valid: false}}, + {source: nilPointerInt64, expect: Float{Valid: false}}, + {source: nilPointerUint8, expect: Float{Valid: false}}, + {source: nilPointerUint16, expect: Float{Valid: false}}, + {source: nilPointerUint32, expect: Float{Valid: false}}, + {source: nilPointerUint64, expect: Float{Valid: false}}, + {source: nilPointerFloat32, expect: Float{Valid: false}}, + {source: nilPointerFloat64, expect: Float{Valid: false}}, + } + + for _, bitWidth := range []uint8{8, 16, 32, 64} { + bitWidth := bitWidth + t.Run(strconv.Itoa(int(bitWidth)), func(t *testing.T) { + t.Parallel() + + for i, tt := range successfulTests { + r := Float{BitWidth: bitWidth} + if err := r.Set(tt.source); err != nil { + t.Errorf("%d: %v", i, err) + } + + tt.expect.BitWidth = bitWidth + if !r.Equal(&tt.expect) { + t.Errorf("%d: %v != %v", i, r, tt.expect) + } + } + }) + } +} + +func TestFloat64Set(t *testing.T) { + successfulTests := []struct { + source any + result Float + }{ + {source: float32(1), result: Float{Value: 1, Valid: true}}, + {source: float64(1), result: Float{Value: 1, Valid: true}}, + {source: int8(1), result: Float{Value: 1, Valid: true}}, + {source: int16(1), result: Float{Value: 1, Valid: true}}, + {source: int32(1), result: Float{Value: 1, Valid: true}}, + {source: int64(1), result: Float{Value: 1, Valid: true}}, + {source: int8(-1), result: Float{Value: -1, Valid: true}}, + {source: int16(-1), result: Float{Value: -1, Valid: true}}, + {source: int32(-1), result: Float{Value: -1, Valid: true}}, + {source: int64(-1), result: Float{Value: -1, Valid: true}}, + {source: uint8(1), result: Float{Value: 1, Valid: true}}, + {source: uint16(1), result: Float{Value: 1, Valid: true}}, + {source: uint32(1), result: Float{Value: 1, Valid: true}}, + {source: uint64(1), result: Float{Value: 1, Valid: true}}, + {source: "1", result: Float{Value: 1, Valid: true}}, + {source: _int8(1), result: Float{Value: 1, Valid: true}}, + {source: &Float{Value: 1, Valid: true}, result: Float{Value: 1, Valid: true}}, + {source: nilPointerInt8, result: Float{Valid: false}}, + {source: nilPointerInt16, result: Float{Valid: false}}, + {source: nilPointerInt32, result: Float{Valid: false}}, + {source: nilPointerInt64, result: Float{Valid: false}}, + {source: nilPointerUint8, result: Float{Valid: false}}, + {source: nilPointerUint16, result: Float{Valid: false}}, + {source: nilPointerUint32, result: Float{Valid: false}}, + {source: nilPointerUint64, result: Float{Valid: false}}, + {source: nilPointerFloat32, result: Float{Valid: false}}, + {source: nilPointerFloat64, result: Float{Valid: false}}, + } + + for i, tt := range successfulTests { + var r Float + err := r.Set(tt.source) + if err != nil { + t.Errorf("%d: %v", i, err) + } + + if !r.Equal(&tt.result) { + t.Errorf("%d: %v != %v", i, r, tt.result) + } + } +} + +func TestFloatOverflows(t *testing.T) { + cases := []struct { + source any + bitWidth uint8 + expectError bool + }{ + {source: uint64(math.MaxInt32), bitWidth: 64, expectError: false}, + {source: uint64(math.MaxInt32 + 1), bitWidth: 64, expectError: false}, + {source: uint64(math.MaxInt64), bitWidth: 64, expectError: true}, + {source: uint64(math.MaxUint32), bitWidth: 64, expectError: false}, + {source: uint64(math.MaxUint64), bitWidth: 64, expectError: true}, + {source: int64(math.MinInt64), bitWidth: 64, expectError: true}, + {source: int64(math.MinInt32), bitWidth: 64, expectError: false}, + {source: int64(math.MinInt32 - 1), bitWidth: 64, expectError: false}, + + {source: uint64(math.MaxInt16), bitWidth: 32, expectError: false}, + {source: uint64(math.MaxInt16 + 1), bitWidth: 32, expectError: false}, + {source: uint64(math.MaxInt32), bitWidth: 32, expectError: true}, + {source: uint64(math.MaxUint16), bitWidth: 32, expectError: false}, + {source: uint64(math.MaxUint32), bitWidth: 32, expectError: true}, + {source: int64(math.MinInt32), bitWidth: 32, expectError: true}, + {source: int64(math.MinInt16), bitWidth: 32, expectError: false}, + {source: int64(math.MinInt16 - 1), bitWidth: 32, expectError: false}, + } + for idx, tc := range cases { + tc := tc + t.Run(strconv.Itoa(idx), func(t *testing.T) { + t.Parallel() + + r := Float{BitWidth: tc.bitWidth} + err := r.Set(tc.source) + if tc.expectError { + assert.Errorf(t, err, "with %T %#v", tc.source, tc.source) + } else { + assert.NoErrorf(t, err, "with %T %#v", tc.source, tc.source) + } + }) + } +} diff --git a/plugin-sdk/scalar/inet.go b/plugin-sdk/scalar/inet.go new file mode 100644 index 00000000..953ee772 --- /dev/null +++ b/plugin-sdk/scalar/inet.go @@ -0,0 +1,150 @@ +package scalar + +import ( + "encoding" + "fmt" + "net" + "strings" + + "github.com/apache/arrow/go/v16/arrow" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" +) + +type Inet struct { + Valid bool + Value *net.IPNet +} + +func (s *Inet) IsValid() bool { + return s.Valid +} + +func (*Inet) DataType() arrow.DataType { + return types.ExtensionTypes.Inet +} + +func (s *Inet) Equal(rhs Scalar) bool { + if rhs == nil { + return false + } + r, ok := rhs.(*Inet) + if !ok { + return false + } + return s.Valid == r.Valid && s.Value.String() == r.Value.String() +} + +func (s *Inet) String() string { + if !s.Valid { + return nullValueStr + } + return s.Value.String() +} + +func (s *Inet) Get() any { + return s.Value +} + +func (s *Inet) Set(val any) error { + if val == nil { + return nil + } + + if sc, ok := val.(Scalar); ok { + if !sc.IsValid() { + s.Valid = false + return nil + } + return s.Set(sc.Get()) + } + + switch value := val.(type) { + case net.IPNet: + s.Value = &value + case net.IP: + if len(value) == 0 { + return nil + } + bitCount := len(value) * 8 + mask := net.CIDRMask(bitCount, bitCount) + s.Value = &net.IPNet{Mask: mask, IP: value} + case string: + ip, ipnet, err := net.ParseCIDR(value) + if err != nil { + ip := net.ParseIP(value) + if ip == nil { + return &ValidationError{Type: types.ExtensionTypes.Inet, Msg: "cannot parse string as IP", Value: value} + } + + if ipv4 := maybeGetIPv4(value, ip); ipv4 != nil { + ipnet = &net.IPNet{IP: ipv4, Mask: net.CIDRMask(32, 32)} + } else { + ipnet = &net.IPNet{IP: ip, Mask: net.CIDRMask(128, 128)} + } + } else { + ipnet.IP = ip + if ipv4 := maybeGetIPv4(value, ipnet.IP); ipv4 != nil { + ipnet.IP = ipv4 + if len(ipnet.Mask) == 16 { + ipnet.Mask = ipnet.Mask[12:] // Not sure this is ever needed. + } + } + } + s.Value = ipnet + case *net.IPNet: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *net.IP: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *string: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + default: + if tv, ok := value.(encoding.TextMarshaler); ok { + text, err := tv.MarshalText() + if err != nil { + return &ValidationError{Type: types.ExtensionTypes.UUID, Msg: "cannot marshal text", Err: err, Value: val} + } + return s.Set(string(text)) + } + if sv, ok := value.(fmt.Stringer); ok { + return s.Set(sv.String()) + } + if originalSrc, ok := underlyingPtrType(val); ok { + return s.Set(originalSrc) + } + return &ValidationError{Type: types.ExtensionTypes.UUID, Msg: noConversion, Value: value} + } + s.Valid = true + return nil +} + +// Convert the net.IP to IPv4, if appropriate. +// +// When parsing a string to a net.IP using net.ParseIP() and the like, we get a +// 16 byte slice for IPv4 addresses as well as IPv6 addresses. This function +// calls To4() to convert them to a 4 byte slice. This is useful as it allows +// users of the net.IP check for IPv4 addresses based on the length and makes +// it clear we are handling IPv4 as opposed to IPv6 or IPv4-mapped IPv6 +// addresses. +func maybeGetIPv4(input string, ip net.IP) net.IP { + // Do not do this if the provided input looks like IPv6. This is because + // To4() on IPv4-mapped IPv6 addresses converts them to IPv4, which behave + // different in some cases. + if strings.Contains(input, ":") { + return nil + } + + return ip.To4() +} diff --git a/plugin-sdk/scalar/inet_test.go b/plugin-sdk/scalar/inet_test.go new file mode 100644 index 00000000..231cfcef --- /dev/null +++ b/plugin-sdk/scalar/inet_test.go @@ -0,0 +1,118 @@ +package scalar + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "net" + "strings" + "testing" +) + +type textMarshaler struct { + Text string +} + +func (t textMarshaler) MarshalText() (text []byte, err error) { + return []byte(t.Text), err +} + +// nolint:unparam +func mustParseCIDR(t testing.TB, s string) *net.IPNet { + _, ipnet, err := net.ParseCIDR(s) + if err != nil { + t.Fatal(err) + } + + return ipnet +} + +func mustParseInet(t testing.TB, s string) *net.IPNet { + ip, ipnet, err := net.ParseCIDR(s) + if err == nil { + if ipv4 := ip.To4(); ipv4 != nil { + ipnet.IP = ipv4 + } + return ipnet + } + + // May be bare IP address. + // + ip = net.ParseIP(s) + if ip == nil { + t.Fatal(errors.New("unable to parse inet address")) + } + ipnet = &net.IPNet{IP: ip, Mask: net.CIDRMask(128, 128)} + if ipv4 := ip.To4(); ipv4 != nil { + ipnet.IP = ipv4 + ipnet.Mask = net.CIDRMask(32, 32) + } + return ipnet +} + +func TestInetSet(t *testing.T) { + successfulTests := []struct { + source any + result Inet + }{ + {source: mustParseCIDR(t, "127.0.0.1/32"), result: Inet{Value: mustParseCIDR(t, "127.0.0.1/32"), Valid: true}}, + {source: mustParseCIDR(t, "127.0.0.1/32").IP, result: Inet{Value: mustParseCIDR(t, "127.0.0.1/32"), Valid: true}}, + {source: "127.0.0.1/32", result: Inet{Value: mustParseCIDR(t, "127.0.0.1/32"), Valid: true}}, + {source: "1.2.3.4/24", result: Inet{Value: &net.IPNet{IP: net.ParseIP("1.2.3.4").To4(), Mask: net.CIDRMask(24, 32)}, Valid: true}}, + {source: "10.0.0.1", result: Inet{Value: mustParseInet(t, "10.0.0.1"), Valid: true}}, + {source: "2607:f8b0:4009:80b::200e", result: Inet{Value: mustParseInet(t, "2607:f8b0:4009:80b::200e"), Valid: true}}, + {source: net.ParseIP(""), result: Inet{}}, + {source: "0.0.0.0/8", result: Inet{Value: mustParseInet(t, "0.0.0.0/8"), Valid: true}}, + {source: "::ffff:0.0.0.0/104", result: Inet{Value: &net.IPNet{IP: net.ParseIP("::ffff:0.0.0.0"), Mask: net.CIDRMask(104, 128)}, Valid: true}}, + {source: textMarshaler{"127.0.0.1"}, result: Inet{Value: mustParseInet(t, "127.0.0.1"), Valid: true}}, + {source: func(s string) fmt.Stringer { + var b strings.Builder + b.WriteString(s) + return &b + }("127.0.0.1"), result: Inet{Value: mustParseInet(t, "127.0.0.1"), Valid: true}}, + {source: &Inet{Value: &net.IPNet{IP: net.ParseIP("::ffff:0.0.0.0"), Mask: net.CIDRMask(104, 128)}, Valid: true}, result: Inet{Value: &net.IPNet{IP: net.ParseIP("::ffff:0.0.0.0"), Mask: net.CIDRMask(104, 128)}, Valid: true}}, + {source: (*net.IPNet)(nil), result: Inet{Value: nil, Valid: false}}, + } + + for i, tt := range successfulTests { + var r Inet + err := r.Set(tt.source) + if err != nil { + t.Errorf("%d: %v", i, err) + continue + } + + if !r.Equal(&tt.result) { + t.Errorf("%d: %v != %v", i, r, tt.result) + } + } +} + +func TestInetMarshalUnmarshal(t *testing.T) { + var r Inet + err := r.Set("10.244.0.0/24") + if err != nil { + t.Fatal(err) + } + b, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } + var r2 Inet + err = json.Unmarshal(b, &r2) + if err != nil { + t.Fatal(err) + } + if !r.Equal(&r2) { + t.Errorf("%v != %v", r, r2) + } + + // workaround this Golang bug: https://github.com/golang/go/issues/35727 + if !bytes.Equal(r.Value.Mask, r2.Value.Mask) { + t.Errorf("%v != %v", r.Value.Mask, r2.Value.Mask) + } + if !net.IP.Equal(r.Value.IP, r2.Value.IP) { + t.Errorf("%v != %v", r.Value.IP, r2.Value.IP) + } +} diff --git a/plugin-sdk/scalar/int.go b/plugin-sdk/scalar/int.go new file mode 100644 index 00000000..46f86858 --- /dev/null +++ b/plugin-sdk/scalar/int.go @@ -0,0 +1,253 @@ +package scalar + +import ( + "math" + "strconv" + + "github.com/apache/arrow/go/v16/arrow" +) + +type Int struct { + Valid bool + Value int64 + BitWidth uint8 // defaults to 64 +} + +func (s *Int) IsValid() bool { + return s.Valid +} + +func (s *Int) DataType() arrow.DataType { + switch s.getBitWidth() { + case 64: + return arrow.PrimitiveTypes.Int64 + case 32: + return arrow.PrimitiveTypes.Int32 + case 16: + return arrow.PrimitiveTypes.Int16 + case 8: + return arrow.PrimitiveTypes.Int8 + default: + panic("invalid bit width") + } +} + +func (s *Int) String() string { + if !s.Valid { + return nullValueStr + } + return strconv.FormatInt(s.Value, 10) +} + +func (s *Int) Equal(rhs Scalar) bool { + if rhs == nil { + return false + } + r, ok := rhs.(*Int) + if !ok { + return false + } + return s.getBitWidth() == r.getBitWidth() && s.Valid == r.Valid && s.Value == r.Value +} + +func (s *Int) Get() any { + return s.Value +} + +func (s *Int) Set(val any) error { + if val == nil { + s.Valid = false + return nil + } + + if sc, ok := val.(Scalar); ok { + if !sc.IsValid() { + s.Valid = false + return nil + } + return s.Set(sc.Get()) + } + + switch value := val.(type) { + case int8: + v := int64(value) + if err := s.validateValue(v); err != nil { + return err + } + s.Value = v + case int16: + v := int64(value) + if err := s.validateValue(v); err != nil { + return err + } + s.Value = v + case int32: + v := int64(value) + if err := s.validateValue(v); err != nil { + return err + } + s.Value = v + case int64: + if err := s.validateValue(value); err != nil { + return err + } + s.Value = value + case int: + v := int64(value) + if err := s.validateValue(v); err != nil { + return err + } + s.Value = v + case uint8: + v := int64(value) + if err := s.validateValue(v); err != nil { + return err + } + s.Value = v + case uint16: + v := int64(value) + if err := s.validateValue(v); err != nil { + return err + } + s.Value = v + case uint32: + v := int64(value) + if err := s.validateValue(v); err != nil { + return err + } + s.Value = v + case uint64: + return s.Set(int64(value)) + case uint: + return s.Set(int64(value)) + case float32: + return s.Set(int64(value)) + case float64: + return s.Set(int64(value)) + case string: + v, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return &ValidationError{Type: s.DataType(), Msg: "invalid string", Value: value} + } + return s.Set(v) + case *int8: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *int16: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *int32: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *int64: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *int: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *uint8: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *uint16: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *uint32: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *uint64: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *uint: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *float32: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *float64: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *string: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + default: + if originalSrc, ok := underlyingNumberType(value); ok { + return s.Set(originalSrc) + } + return &ValidationError{Type: s.DataType(), Msg: noConversion, Value: value} + } + s.Valid = true + return nil +} + +func (s *Int) validateValue(value int64) error { + switch s.getBitWidth() { + case 8: + if value > math.MaxInt8 { + return &ValidationError{Type: s.DataType(), Msg: "value greater than MaxInt8", Value: value} + } + if value < math.MinInt8 { + return &ValidationError{Type: s.DataType(), Msg: "value less than MinInt8", Value: value} + } + case 16: + if value > math.MaxInt16 { + return &ValidationError{Type: s.DataType(), Msg: "value greater than MaxInt16", Value: value} + } + if value < math.MinInt16 { + return &ValidationError{Type: s.DataType(), Msg: "value less than MinInt16", Value: value} + } + case 32: + if value > math.MaxInt32 { + return &ValidationError{Type: s.DataType(), Msg: "value greater than MaxInt32", Value: value} + } + if value < math.MinInt32 { + return &ValidationError{Type: s.DataType(), Msg: "value less than MinInt32", Value: value} + } + } + return nil +} + +func (s *Int) getBitWidth() uint8 { + if s.BitWidth == 0 { + return 64 // default + } + return s.BitWidth +} diff --git a/plugin-sdk/scalar/int_test.go b/plugin-sdk/scalar/int_test.go new file mode 100644 index 00000000..c30c33fe --- /dev/null +++ b/plugin-sdk/scalar/int_test.go @@ -0,0 +1,106 @@ +package scalar + +import ( + "math" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestInt8Set(t *testing.T) { + successfulTests := []struct { + source any + expect Int + }{ + {source: int8(1), expect: Int{Value: 1, Valid: true}}, + {source: int16(1), expect: Int{Value: 1, Valid: true}}, + {source: int32(1), expect: Int{Value: 1, Valid: true}}, + {source: int64(1), expect: Int{Value: 1, Valid: true}}, + {source: int8(-1), expect: Int{Value: -1, Valid: true}}, + {source: int16(-1), expect: Int{Value: -1, Valid: true}}, + {source: int32(-1), expect: Int{Value: -1, Valid: true}}, + {source: int64(-1), expect: Int{Value: -1, Valid: true}}, + {source: uint8(1), expect: Int{Value: 1, Valid: true}}, + {source: uint16(1), expect: Int{Value: 1, Valid: true}}, + {source: uint32(1), expect: Int{Value: 1, Valid: true}}, + {source: uint64(1), expect: Int{Value: 1, Valid: true}}, + {source: float32(1), expect: Int{Value: 1, Valid: true}}, + {source: float64(1), expect: Int{Value: 1, Valid: true}}, + {source: "1", expect: Int{Value: 1, Valid: true}}, + {source: _int8(1), expect: Int{Value: 1, Valid: true}}, + {source: &Int{Value: 1, Valid: true}, expect: Int{Value: 1, Valid: true}}, + } + + for _, bitWidth := range []uint8{8, 16, 32, 64} { + bitWidth := bitWidth + t.Run(strconv.Itoa(int(bitWidth)), func(t *testing.T) { + t.Parallel() + + for i, tt := range successfulTests { + r := Int{BitWidth: bitWidth} + err := r.Set(tt.source) + if err != nil { + t.Errorf("%d: %v", i, err) + } + + tt.expect.BitWidth = bitWidth + if !r.Equal(&tt.expect) { + t.Errorf("%d: %v != %v", i, r, tt.expect) + } + } + }) + } +} + +func TestIntOverflows(t *testing.T) { + cases := []struct { + source any + bitWidth uint8 + expectError bool + }{ + {source: int16(math.MaxInt8), bitWidth: 8, expectError: false}, + {source: int16(math.MaxInt8 + 1), bitWidth: 8, expectError: true}, + {source: int16(math.MinInt8), bitWidth: 8, expectError: false}, + {source: int16(math.MinInt8 - 1), bitWidth: 8, expectError: true}, + {source: uint16(math.MaxInt8), bitWidth: 8, expectError: false}, + {source: float32(math.MaxInt8), bitWidth: 8, expectError: false}, + {source: float32(math.MaxInt8 + 1), bitWidth: 8, expectError: true}, + {source: float32(math.MinInt8 - 1), bitWidth: 8, expectError: true}, + {source: &Int{Value: math.MaxInt8 + 1, Valid: true}, bitWidth: 8, expectError: true}, + + {source: int32(math.MaxInt16), bitWidth: 16, expectError: false}, + {source: int32(math.MaxInt16 + 1), bitWidth: 16, expectError: true}, + {source: int32(math.MinInt16), bitWidth: 16, expectError: false}, + {source: int32(math.MinInt16 - 1), bitWidth: 16, expectError: true}, + {source: uint16(math.MaxInt16), bitWidth: 16, expectError: false}, + {source: float32(math.MaxInt16), bitWidth: 16, expectError: false}, + {source: float32(math.MaxInt16 + 1), bitWidth: 16, expectError: true}, + {source: float32(math.MinInt16 - 1), bitWidth: 16, expectError: true}, + + {source: int64(math.MaxInt32), bitWidth: 32, expectError: false}, + {source: int64(math.MaxInt32 + 1), bitWidth: 32, expectError: true}, + {source: int64(math.MinInt32), bitWidth: 32, expectError: false}, + {source: int64(math.MinInt32 - 1), bitWidth: 32, expectError: true}, + {source: uint32(math.MaxInt32), bitWidth: 32, expectError: false}, + {source: uint64(math.MaxInt32), bitWidth: 32, expectError: false}, + {source: uint64(math.MaxInt32 + 1), bitWidth: 32, expectError: true}, + {source: float64(math.MaxInt32), bitWidth: 32, expectError: false}, + {source: float64(math.MaxInt32 + 1), bitWidth: 32, expectError: true}, + {source: float64(math.MinInt32 - 1), bitWidth: 32, expectError: true}, + } + for idx, tc := range cases { + tc := tc + t.Run(strconv.Itoa(idx), func(t *testing.T) { + t.Parallel() + + r := Int{BitWidth: tc.bitWidth} + err := r.Set(tc.source) + if tc.expectError { + assert.Errorf(t, err, "with %T %#v", tc.source, tc.source) + } else { + assert.NoErrorf(t, err, "with %T %#v", tc.source, tc.source) + } + }) + } +} diff --git a/plugin-sdk/scalar/interval.go b/plugin-sdk/scalar/interval.go new file mode 100644 index 00000000..6101aa8f --- /dev/null +++ b/plugin-sdk/scalar/interval.go @@ -0,0 +1,264 @@ +package scalar + +import ( + "encoding/json" + + "github.com/apache/arrow/go/v16/arrow" +) + +type MonthInterval struct { + Int +} + +type monthIntervalData struct { + Months int32 `json:"months"` +} + +func (*MonthInterval) DataType() arrow.DataType { + return arrow.FixedWidthTypes.MonthInterval +} + +func (s *MonthInterval) Equal(rhs Scalar) bool { + if rhs == nil { + return false + } + r, ok := rhs.(*MonthInterval) + if !ok { + return false + } + return s.Valid == r.Valid && s.Value == r.Value +} + +func (s *MonthInterval) String() string { + if !s.Valid { + return nullValueStr + } + + b, _ := json.Marshal(monthIntervalData{Months: int32(s.Value)}) + return string(b) +} + +func (s *MonthInterval) Set(value any) error { + if mi, ok := value.(arrow.MonthInterval); ok { + return s.Int.Set(int32(mi)) + } + + switch v := value.(type) { + case string: + if len(v) == 0 { + s.Valid = false + return nil + } + return s.Int.Set(value) + case []byte: + if len(v) == 0 { + s.Valid = false + return nil + } + + var mi monthIntervalData + if err := json.Unmarshal(v, &mi); err != nil { + return err + } + s.Valid = true + s.Value = int64(mi.Months) + return nil + case *string: + if v == nil { + s.Valid = false + return nil + } + return s.Set(*v) + case map[string]any: + b, _ := json.Marshal(v) + return s.Set(b) + default: + return s.Int.Set(value) + } +} + +type DayTimeInterval struct { + Value arrow.DayTimeInterval + Valid bool +} + +func (*DayTimeInterval) DataType() arrow.DataType { + return arrow.FixedWidthTypes.DayTimeInterval +} + +func (s *DayTimeInterval) IsValid() bool { + return s.Valid +} + +func (s *DayTimeInterval) String() string { + if !s.Valid { + return nullValueStr + } + b, _ := json.Marshal(s.Value) + return string(b) +} + +func (s *DayTimeInterval) Equal(rhs Scalar) bool { + if rhs == nil { + return false + } + r, ok := rhs.(*DayTimeInterval) + if !ok { + return false + } + return s.Valid == r.Valid && s.Value == r.Value +} + +func (s *DayTimeInterval) Set(value any) error { + if value == nil { + s.Valid = false + return nil + } + + if dti, ok := value.(arrow.DayTimeInterval); ok { + s.Valid = true + s.Value = dti + return nil + } + + switch v := value.(type) { + case string: + if len(v) == 0 { + s.Valid = false + return nil + } + + var dti arrow.DayTimeInterval + if err := json.Unmarshal([]byte(v), &dti); err != nil { + return err + } + s.Valid = true + s.Value = dti + return nil + case []byte: + if len(v) == 0 { + s.Valid = false + return nil + } + + var dti arrow.DayTimeInterval + if err := json.Unmarshal(v, &dti); err != nil { + return err + } + s.Valid = true + s.Value = dti + return nil + case *string: + if v == nil { + s.Valid = false + return nil + } + return s.Set(*v) + case map[string]any: + b, _ := json.Marshal(v) + return s.Set(b) + default: + return &ValidationError{Type: s.DataType(), Msg: noConversion, Value: value} + } +} + +func (s *DayTimeInterval) Get() any { + if !s.Valid { + return nil + } + + return s.Value +} + +type MonthDayNanoInterval struct { + Value arrow.MonthDayNanoInterval + Valid bool +} + +func (*MonthDayNanoInterval) DataType() arrow.DataType { + return arrow.FixedWidthTypes.MonthDayNanoInterval +} + +func (s *MonthDayNanoInterval) IsValid() bool { + return s.Valid +} + +func (s *MonthDayNanoInterval) String() string { + if !s.Valid { + return nullValueStr + } + b, _ := json.Marshal(s.Value) + return string(b) +} + +func (s *MonthDayNanoInterval) Equal(rhs Scalar) bool { + if rhs == nil { + return false + } + r, ok := rhs.(*MonthDayNanoInterval) + if !ok { + return false + } + return s.Valid == r.Valid && s.Value == r.Value +} + +func (s *MonthDayNanoInterval) Set(value any) error { + if value == nil { + s.Valid = false + return nil + } + + if dti, ok := value.(arrow.MonthDayNanoInterval); ok { + s.Valid = true + s.Value = dti + return nil + } + + switch v := value.(type) { + case string: + if len(v) == 0 { + s.Valid = false + return nil + } + + var dti arrow.MonthDayNanoInterval + if err := json.Unmarshal([]byte(v), &dti); err != nil { + return err + } + s.Valid = true + s.Value = dti + return nil + case []byte: + if len(v) == 0 { + s.Valid = false + return nil + } + + var dti arrow.MonthDayNanoInterval + if err := json.Unmarshal(v, &dti); err != nil { + return err + } + s.Valid = true + s.Value = dti + return nil + case *string: + if v == nil { + s.Valid = false + return nil + } + return s.Set(*v) + case map[string]any: + b, _ := json.Marshal(v) + return s.Set(b) + default: + return &ValidationError{Type: s.DataType(), Msg: noConversion, Value: value} + } +} + +func (s *MonthDayNanoInterval) Get() any { + if !s.Valid { + return nil + } + + return s.Value +} diff --git a/plugin-sdk/scalar/json.go b/plugin-sdk/scalar/json.go new file mode 100644 index 00000000..c12e8b97 --- /dev/null +++ b/plugin-sdk/scalar/json.go @@ -0,0 +1,171 @@ +package scalar + +import ( + "bytes" + "encoding/json" + "reflect" + + "github.com/apache/arrow/go/v16/arrow" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" +) + +type JSON struct { + Valid bool + Value []byte +} + +func (s *JSON) IsValid() bool { + return s.Valid +} + +func (*JSON) DataType() arrow.DataType { + return types.ExtensionTypes.JSON +} + +func (s *JSON) Get() any { + return s.Value +} + +func (s *JSON) Equal(rhs Scalar) bool { + if rhs == nil { + return false + } + r, ok := rhs.(*JSON) + if !ok { + return false + } + if !s.Valid && !r.Valid { + return true + } + + if s.Valid != r.Valid { + return false + } + + equal, err := jsonBytesEqual(s.Value, r.Value) + if err != nil { + return false + } + return equal +} + +func (s *JSON) String() string { + if !s.Valid { + return nullValueStr + } + return string(s.Value) +} + +func (s *JSON) Set(val any) error { + if val == nil { + return nil + } + + if sc, ok := val.(Scalar); ok { + if !sc.IsValid() { + s.Valid = false + return nil + } + return s.Set(sc.Get()) + } + + switch value := val.(type) { + case string: + if value == "" { + return nil + } + if !json.Valid([]byte(value)) { + return &ValidationError{Type: types.ExtensionTypes.JSON, Msg: "invalid json string", Value: value} + } + s.Value = []byte(value) + case *string: + if value == nil { + return nil + } + return s.Set(*value) + case []byte: + if value == nil { + return nil + } + if string(value) == "" { + return nil + } + + if !json.Valid(value) { + return &ValidationError{Type: types.ExtensionTypes.UUID, Msg: "invalid json byte array", Value: value} + } + s.Value = value + // Encode* methods are defined on *JSON. If JSON is passed directly then the + // struct itself would be encoded instead of Bytes. This is clearly a footgun + // so detect and return an error. See https://github.com/jackc/pgx/issues/350. + case JSON: + return &ValidationError{Type: types.ExtensionTypes.JSON, Msg: "use pointer to JSON instead of value", Value: value} + default: + buffer := &bytes.Buffer{} + encoder := json.NewEncoder(buffer) + encoder.SetEscapeHTML(false) + err := encoder.Encode(value) + if err != nil { + return err + } + + // JSON encoder adds a newline to the end of the output that we don't want. + buf := bytes.TrimSuffix(buffer.Bytes(), []byte("\n")) + // For map and slice jsons, it is easier for users to work with '[]' or '{}' instead of JSON's 'null'. + if bytes.Equal(buf, []byte(`null`)) { + if isEmptyStringMap(value) { + s.Value = []byte("{}") + s.Valid = true + return nil + } + + if isEmptySlice(value) { + s.Value = []byte("[]") + s.Valid = true + return nil + } + } + s.Value = buf + } + s.Valid = true + return nil +} + +// isEmptyStringMap returns true if the value is a map from string to any (i.e. map[string]any). +// We need to use reflection for this, because it impossible to type-assert a map[string]string into a +// map[string]any. See https://go.dev/doc/faq#convert_slice_of_interface. +func isEmptyStringMap(value any) bool { + if reflect.TypeOf(value).Kind() != reflect.Map { + return false + } + + if reflect.TypeOf(value).Key().Kind() != reflect.String { + return false + } + + return reflect.ValueOf(value).Len() == 0 +} + +// isEmptySlice returns true if the value is a slice (i.e. []any). +// We need to use reflection for this, because it impossible to type-assert a map[string]string into a +// map[string]any. See https://go.dev/doc/faq#convert_slice_of_interface. +func isEmptySlice(value any) bool { + if reflect.TypeOf(value).Kind() != reflect.Slice { + return false + } + + return reflect.ValueOf(value).Len() == 0 +} + +// JSONBytesEqual compares the JSON in two byte slices. +func jsonBytesEqual(a, b []byte) (bool, error) { + var j, j2 any + if err := json.Unmarshal(a, &j); err != nil { + return false, err + } + if err := json.Unmarshal(b, &j2); err != nil { + return false, err + } + return reflect.DeepEqual(j2, j), nil +} diff --git a/plugin-sdk/scalar/json_test.go b/plugin-sdk/scalar/json_test.go new file mode 100644 index 00000000..36fe4c2d --- /dev/null +++ b/plugin-sdk/scalar/json_test.go @@ -0,0 +1,60 @@ +package scalar + +import "testing" + +type Foo struct { + Num int +} + +func TestJSONSet(t *testing.T) { + successfulTests := []struct { + source any + result JSON + }{ + {source: "", result: JSON{Value: []byte("")}}, + {source: "{}", result: JSON{Value: []byte("{}"), Valid: true}}, + {source: `"test"`, result: JSON{Value: []byte(`"test"`), Valid: true}}, + {source: "1", result: JSON{Value: []byte("1"), Valid: true}}, + {source: "[1, 2, 3]", result: JSON{Value: []byte("[1, 2, 3]"), Valid: true}}, + {source: []byte("{}"), result: JSON{Value: []byte("{}"), Valid: true}}, + {source: []byte(`"test"`), result: JSON{Value: []byte(`"test"`), Valid: true}}, + {source: []byte("1"), result: JSON{Value: []byte("1"), Valid: true}}, + {source: []byte("[1, 2, 3]"), result: JSON{Value: []byte("[1, 2, 3]"), Valid: true}}, + {source: ([]byte)(nil), result: JSON{}}, + {source: (*string)(nil), result: JSON{}}, + + {source: []int{1, 2, 3}, result: JSON{Value: []byte("[1,2,3]"), Valid: true}}, + {source: []int(nil), result: JSON{Value: []byte(`[]`), Valid: true}}, + {source: []int{}, result: JSON{Value: []byte(`[]`), Valid: true}}, + {source: []Foo(nil), result: JSON{Value: []byte(`[]`), Valid: true}}, + {source: []Foo{}, result: JSON{Value: []byte(`[]`), Valid: true}}, + {source: []Foo{{1}}, result: JSON{Value: []byte(`[{"Num":1}]`), Valid: true}}, + + {source: map[string]any{"foo": "bar"}, result: JSON{Value: []byte(`{"foo":"bar"}`), Valid: true}}, + {source: map[string]any(nil), result: JSON{Value: []byte(`{}`), Valid: true}}, + {source: map[string]any{}, result: JSON{Value: []byte(`{}`), Valid: true}}, + {source: map[string]string{"foo": "bar"}, result: JSON{Value: []byte(`{"foo":"bar"}`), Valid: true}}, + {source: map[string]string(nil), result: JSON{Value: []byte(`{}`), Valid: true}}, + {source: map[string]string{}, result: JSON{Value: []byte(`{}`), Valid: true}}, + {source: map[string]Foo{"foo": {1}}, result: JSON{Value: []byte(`{"foo":{"Num":1}}`), Valid: true}}, + {source: map[string]Foo(nil), result: JSON{Value: []byte(`{}`), Valid: true}}, + {source: map[string]Foo{}, result: JSON{Value: []byte(`{}`), Valid: true}}, + + {source: nil, result: JSON{}}, + + {source: map[string]any{"test1": "a&b", "test2": "😀"}, result: JSON{Value: []byte(`{"test1": "a&b", "test2": "😀"}`), Valid: true}}, + {source: &JSON{Value: []byte(`{"test1": "a&b", "test2": "😀"}`), Valid: true}, result: JSON{Value: []byte(`{"test1": "a&b", "test2": "😀"}`), Valid: true}}, + } + + for i, tt := range successfulTests { + var d JSON + err := d.Set(tt.source) + if err != nil { + t.Errorf("%d: %v", i, err) + } + + if !d.Equal(&tt.result) { + t.Errorf("%d: %v != %v", i, d, tt.result) + } + } +} diff --git a/plugin-sdk/scalar/list.go b/plugin-sdk/scalar/list.go new file mode 100644 index 00000000..004da5f4 --- /dev/null +++ b/plugin-sdk/scalar/list.go @@ -0,0 +1,167 @@ +package scalar + +import ( + "encoding/json" + "reflect" + "strings" + + "github.com/apache/arrow/go/v16/arrow" +) + +type List struct { + Valid bool + Value Vector + Type arrow.DataType +} + +func (s *List) IsValid() bool { + return s.Valid +} + +func (s *List) DataType() arrow.DataType { + return s.Type +} + +func (s *List) String() string { + if !s.Valid { + return nullValueStr + } + var sb strings.Builder + sb.WriteString("[") + for i, v := range s.Value { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(v.String()) + } + sb.WriteString("]") + return sb.String() +} + +func (s *List) Equal(rhs Scalar) bool { + if rhs == nil { + return false + } + r, ok := rhs.(*List) + if !ok { + return false + } + if s.Valid != r.Valid { + return false + } + if len(s.Value) != len(r.Value) { + return false + } + for i := range s.Value { + if !s.Value[i].Equal(r.Value[i]) { + return false + } + } + return true +} + +func (s *List) Get() any { + return s.Value +} + +func (s *List) Set(val any) error { + if val == nil { + s.Valid = false + return nil + } + if s.Type == nil { + panic("List type is nil") + } + + if sc, ok := val.(Scalar); ok { + if !sc.IsValid() { + s.Valid = false + return nil + } + return s.Set(sc.Get()) + } + + reflectedValue := reflect.ValueOf(val) + for reflectedValue.Kind() == reflect.Pointer && !reflectedValue.IsNil() { + reflectedValue = reflectedValue.Elem() + } + + if !reflectedValue.IsValid() || reflectedValue.IsZero() { + return nil + } + + switch value := val.(type) { + case string: + var x []any + if err := json.Unmarshal([]byte(value), &x); err != nil { + return err + } + length := len(x) + s.Value = make(Vector, length) + for i := 0; i < length; i++ { + s.Value[i] = NewScalar(s.Type.(*arrow.ListType).Elem()) + if x[i] == nil { + continue + } + if err := s.Value[i].Set(x[i]); err != nil { + return err + } + } + + case []byte: + var x []any + if err := json.Unmarshal(value, &x); err != nil { + return err + } + length := len(x) + s.Value = make(Vector, length) + for i := 0; i < length; i++ { + s.Value[i] = NewScalar(s.Type.(*arrow.ListType).Elem()) + if x[i] == nil { + continue + } + if err := s.Value[i].Set(x[i]); err != nil { + return err + } + } + + case *string: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + } + + switch reflectedValue.Kind() { + case reflect.Array, reflect.Slice: + length := reflectedValue.Len() + s.Value = make(Vector, length) + for i := 0; i < length; i++ { + s.Value[i] = NewScalar(s.Type.(*arrow.ListType).Elem()) + iVal := reflectedValue.Index(i) + if isReflectValueNil(iVal) { + continue + } + if err := s.Value[i].Set(iVal.Interface()); err != nil { + return err + } + } + } + + s.Valid = true + return nil +} + +func isReflectValueNil(v reflect.Value) bool { + switch v.Kind() { + case reflect.Pointer, + reflect.UnsafePointer, + reflect.Map, + reflect.Slice, + reflect.Interface: + return v.IsNil() + default: + return false + } +} diff --git a/plugin-sdk/scalar/list_test.go b/plugin-sdk/scalar/list_test.go new file mode 100644 index 00000000..7bed4941 --- /dev/null +++ b/plugin-sdk/scalar/list_test.go @@ -0,0 +1,72 @@ +package scalar + +import ( + "net" + "testing" + + "github.com/apache/arrow/go/v16/arrow" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" +) + +func TestListSet(t *testing.T) { + ipOne := net.IP{192, 168, 1, 1} + ipNet := net.IPNet{IP: ipOne, Mask: net.IPMask{255, 255, 255, 255}} + typedNil := (*net.IP)(nil) + successfulTests := []struct { + source any + result List + }{ + {source: []int{1, 2}, result: List{Value: []Scalar{ + &Int{Value: 1, Valid: true}, + &Int{Value: 2, Valid: true}, + }, Valid: true, Type: arrow.ListOf(arrow.PrimitiveTypes.Int64)}}, + {source: &List{Value: []Scalar{ + &Int{Value: 1, Valid: true}, + &Int{Value: 2, Valid: true}, + }, Valid: true, Type: arrow.ListOf(arrow.PrimitiveTypes.Int64)}, + result: List{Value: []Scalar{ + &Int{Value: 1, Valid: true}, + &Int{Value: 2, Valid: true}, + }, Valid: true, Type: arrow.ListOf(arrow.PrimitiveTypes.Int64)}}, + {source: []*net.IPNet{&ipNet, nil}, result: List{Value: []Scalar{ + &Inet{Value: &ipNet, Valid: true}, + &Inet{Valid: false}, + }, Valid: true, Type: arrow.ListOf(types.ExtensionTypes.Inet)}}, + {source: []*net.IP{&ipOne, typedNil, nil}, result: List{Value: []Scalar{ + &Inet{Value: &ipNet, Valid: true}, + &Inet{Valid: false}, + &Inet{Valid: false}, + }, Valid: true, Type: arrow.ListOf(types.ExtensionTypes.Inet)}}, + {source: `[1, 2]`, result: List{Value: []Scalar{ + &Int{Value: 1, Valid: true}, + &Int{Value: 2, Valid: true}, + }, Valid: true, Type: arrow.ListOf(arrow.PrimitiveTypes.Int64)}}, + {source: `[1, null, 2]`, result: List{Value: []Scalar{ + &Int{Value: 1, Valid: true}, + &Int{Valid: false}, + &Int{Value: 2, Valid: true}, + }, Valid: true, Type: arrow.ListOf(arrow.PrimitiveTypes.Int64)}}, + {source: &[]net.IPNet{ipNet}, result: List{Value: []Scalar{ + &Inet{Value: &ipNet, Valid: true}, + }, Valid: true, Type: arrow.ListOf(types.ExtensionTypes.Inet)}}, + {source: &[]*net.IPNet{&ipNet, nil}, result: List{Value: []Scalar{ + &Inet{Value: &ipNet, Valid: true}, + &Inet{Valid: false}, + }, Valid: true, Type: arrow.ListOf(types.ExtensionTypes.Inet)}}, + } + + for i, tt := range successfulTests { + r := List{ + Type: tt.result.Type, + } + err := r.Set(tt.source) + if err != nil { + t.Errorf("%d: %v", i, err) + } + + if !r.Equal(&tt.result) { + t.Errorf("%d: %v != %v", i, r, tt.result) + } + } +} diff --git a/plugin-sdk/scalar/mac.go b/plugin-sdk/scalar/mac.go new file mode 100644 index 00000000..9502df33 --- /dev/null +++ b/plugin-sdk/scalar/mac.go @@ -0,0 +1,92 @@ +package scalar + +import ( + "net" + + "github.com/apache/arrow/go/v16/arrow" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" +) + +type Mac struct { + Valid bool + Value net.HardwareAddr +} + +func (s *Mac) IsValid() bool { + return s.Valid +} + +func (*Mac) DataType() arrow.DataType { + return types.ExtensionTypes.MAC +} + +func (s *Mac) String() string { + if !s.Valid { + return nullValueStr + } + return s.Value.String() +} + +func (s *Mac) Equal(rhs Scalar) bool { + if rhs == nil { + return false + } + r, ok := rhs.(*Mac) + if !ok { + return false + } + return s.Valid == r.Valid && s.Value.String() == r.Value.String() +} + +func (s *Mac) Get() any { + return s.Value +} + +func (s *Mac) Set(val any) error { + if val == nil { + s.Valid = false + return nil + } + + if sc, ok := val.(Scalar); ok { + if !sc.IsValid() { + s.Valid = false + return nil + } + return s.Set(sc.Get()) + } + + switch value := val.(type) { + case net.HardwareAddr: + addr := make(net.HardwareAddr, len(value)) + copy(addr, value) + s.Value = addr + case string: + addr, err := net.ParseMAC(value) + if err != nil { + return err + } + s.Value = addr + case *net.HardwareAddr: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *string: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + default: + if originalSrc, ok := underlyingPtrType(value); ok { + return s.Set(originalSrc) + } + return &ValidationError{Type: types.ExtensionTypes.MAC, Msg: noConversion, Value: value} + } + + s.Valid = true + return nil +} diff --git a/plugin-sdk/scalar/mac_test.go b/plugin-sdk/scalar/mac_test.go new file mode 100644 index 00000000..eb6917b3 --- /dev/null +++ b/plugin-sdk/scalar/mac_test.go @@ -0,0 +1,48 @@ +package scalar + +import ( + "net" + "testing" +) + +func TestMacaddrSet(t *testing.T) { + successfulTests := []struct { + source any + result Mac + }{ + { + source: mustParseMacaddr(t, "01:23:45:67:89:ab"), + result: Mac{Value: mustParseMacaddr(t, "01:23:45:67:89:ab"), Valid: true}, + }, + { + source: "01:23:45:67:89:ab", + result: Mac{Value: mustParseMacaddr(t, "01:23:45:67:89:ab"), Valid: true}, + }, + { + source: &Mac{Value: mustParseMacaddr(t, "01:23:45:67:89:ab"), Valid: true}, + result: Mac{Value: mustParseMacaddr(t, "01:23:45:67:89:ab"), Valid: true}, + }, + } + + for i, tt := range successfulTests { + var r Mac + err := r.Set(tt.source) + if err != nil { + t.Errorf("%d: %v", i, err) + } + + if !r.Equal(&tt.result) { + t.Errorf("%d: %v != %v", i, r, tt.result) + } + } +} + +// nolint:unparam +func mustParseMacaddr(t testing.TB, s string) net.HardwareAddr { + addr, err := net.ParseMAC(s) + if err != nil { + t.Fatal(err) + } + + return addr +} diff --git a/plugin-sdk/scalar/scalar.go b/plugin-sdk/scalar/scalar.go new file mode 100644 index 00000000..0be626f3 --- /dev/null +++ b/plugin-sdk/scalar/scalar.go @@ -0,0 +1,279 @@ +package scalar + +import ( + "fmt" + "reflect" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/float16" + "github.com/apache/arrow/go/v16/arrow/memory" + "golang.org/x/exp/maps" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" +) + +// Scalar represents a single value of a specific DataType as opposed to +// an array. +// +// Scalars are useful for passing single value inputs to compute functions +// (not yet implemented) or for representing individual array elements, +// (with a non-trivial cost though). +type Scalar interface { + fmt.Stringer + // IsValid returns true if the value is non-null, otherwise false. + IsValid() bool + // The datatype of the value in this scalar + DataType() arrow.DataType + // Performs cheap validation checks, returns nil if successful + // Validate() error + // tries to set the value of the scalar to the given value + Set(val any) error + + Get() any + Equal(other Scalar) bool +} + +type Vector []Scalar + +func (v Vector) ToArrowRecord(sc *arrow.Schema) arrow.Record { + bldr := array.NewRecordBuilder(memory.DefaultAllocator, sc) + AppendToRecordBuilder(bldr, v) + rec := bldr.NewRecord() + return rec +} + +func (v Vector) Equal(r Vector) bool { + if len(v) != len(r) { + return false + } + for i := range v { + if !v[i].Equal(r[i]) { + return false + } + } + return true +} + +func NewScalar(dt arrow.DataType) Scalar { + switch dt.ID() { + case arrow.TIMESTAMP: + return &Timestamp{Type: dt.(*arrow.TimestampType)} + case arrow.BINARY: + return &Binary{} + case arrow.STRING: + return &String{} + case arrow.LARGE_BINARY: + return &LargeBinary{} + case arrow.LARGE_STRING: + return &LargeString{} + case arrow.INT64: + return &Int{BitWidth: 64} + case arrow.INT32: + return &Int{BitWidth: 32} + case arrow.INT16: + return &Int{BitWidth: 16} + case arrow.INT8: + return &Int{BitWidth: 8} + case arrow.UINT64: + return &Uint{BitWidth: 64} + case arrow.UINT32: + return &Uint{BitWidth: 32} + case arrow.UINT16: + return &Uint{BitWidth: 16} + case arrow.UINT8: + return &Uint{BitWidth: 8} + case arrow.FLOAT64: + return &Float{BitWidth: 64} + case arrow.FLOAT32: + return &Float{BitWidth: 32} + case arrow.FLOAT16: + return &Float{BitWidth: 16} + case arrow.BOOL: + return &Bool{} + case arrow.EXTENSION: + switch { + case arrow.TypeEqual(dt, types.ExtensionTypes.UUID): + return &UUID{} + case arrow.TypeEqual(dt, types.ExtensionTypes.JSON): + return &JSON{} + case arrow.TypeEqual(dt, types.ExtensionTypes.MAC): + return &Mac{} + case arrow.TypeEqual(dt, types.ExtensionTypes.Inet): + return &Inet{} + default: + panic("not implemented extension: " + dt.Name()) + } + case arrow.LIST: + return &List{ + Type: dt, + } + case arrow.DATE64: + return &Date64{} + case arrow.DATE32: + return &Date32{} + case arrow.DURATION: + return &Duration{Int: Int{BitWidth: 64}, Unit: dt.(*arrow.DurationType).Unit} + case arrow.TIME32: + return &Time{ + Int: Int{BitWidth: 32}, + Unit: dt.(*arrow.Time32Type).Unit, + } + case arrow.TIME64: + return &Time{ + Int: Int{BitWidth: 64}, + Unit: dt.(*arrow.Time64Type).Unit, + } + + case arrow.INTERVAL_MONTHS: + return &MonthInterval{Int{BitWidth: 32}} + case arrow.INTERVAL_DAY_TIME: + return &DayTimeInterval{} + case arrow.INTERVAL_MONTH_DAY_NANO: + return &MonthDayNanoInterval{} + + case arrow.STRUCT: + return &Struct{Type: dt.(*arrow.StructType)} + + case arrow.DECIMAL128: + return &Decimal128{Type: dt.(*arrow.Decimal128Type)} + case arrow.DECIMAL256: + return &Decimal256{Type: dt.(*arrow.Decimal256Type)} + + default: + panic("not implemented: " + dt.Name()) + } +} + +func AppendToBuilder(bldr array.Builder, s Scalar) { + if !s.IsValid() { + bldr.AppendNull() + return + } + switch s.DataType().ID() { + case arrow.BINARY: + bldr.(*array.BinaryBuilder).Append(s.(*Binary).Value) + case arrow.LARGE_BINARY: + bldr.(*array.BinaryBuilder).Append(s.(*LargeBinary).Value) + case arrow.STRING: + bldr.(*array.StringBuilder).Append(s.(*String).Value) + case arrow.LARGE_STRING: + bldr.(*array.LargeStringBuilder).Append(s.(*LargeString).s.Value) + case arrow.INT64: + bldr.(*array.Int64Builder).Append(s.(*Int).Value) + case arrow.INT32: + bldr.(*array.Int32Builder).Append(int32(s.(*Int).Value)) + case arrow.INT16: + bldr.(*array.Int16Builder).Append(int16(s.(*Int).Value)) + case arrow.INT8: + bldr.(*array.Int8Builder).Append(int8(s.(*Int).Value)) + case arrow.UINT64: + bldr.(*array.Uint64Builder).Append(s.(*Uint).Value) + case arrow.UINT32: + bldr.(*array.Uint32Builder).Append(uint32(s.(*Uint).Value)) + case arrow.UINT16: + bldr.(*array.Uint16Builder).Append(uint16(s.(*Uint).Value)) + case arrow.UINT8: + bldr.(*array.Uint8Builder).Append(uint8(s.(*Uint).Value)) + case arrow.FLOAT16: + bldr.(*array.Float16Builder).Append(float16.New(float32(s.(*Float).Value))) + case arrow.FLOAT32: + bldr.(*array.Float32Builder).Append(float32(s.(*Float).Value)) + case arrow.FLOAT64: + bldr.(*array.Float64Builder).Append(s.(*Float).Value) + case arrow.BOOL: + bldr.(*array.BooleanBuilder).Append(s.(*Bool).Value) + case arrow.TIMESTAMP: + bldr.(*array.TimestampBuilder).AppendTime(s.(*Timestamp).Value) + case arrow.DURATION: + bldr.(*array.DurationBuilder).Append(arrow.Duration(s.(*Duration).Value)) + case arrow.DATE32: + bldr.(*array.Date32Builder).Append(s.(*Date32).Value) + case arrow.DATE64: + bldr.(*array.Date64Builder).Append(s.(*Date64).Value) + case arrow.TIME32: + bldr.(*array.Time32Builder).Append(arrow.Time32(int32(s.(*Time).Value))) + case arrow.TIME64: + bldr.(*array.Time64Builder).Append(arrow.Time64(s.(*Time).Value)) + case arrow.INTERVAL_MONTHS: + bldr.(*array.MonthIntervalBuilder).Append(arrow.MonthInterval(int32(s.(*MonthInterval).Value))) + case arrow.INTERVAL_DAY_TIME: + bldr.(*array.DayTimeIntervalBuilder).Append(s.(*DayTimeInterval).Value) + case arrow.INTERVAL_MONTH_DAY_NANO: + bldr.(*array.MonthDayNanoIntervalBuilder).Append(s.(*MonthDayNanoInterval).Value) + case arrow.DECIMAL128: + bldr.(*array.Decimal128Builder).Append(s.(*Decimal128).Value) + case arrow.DECIMAL256: + bldr.(*array.Decimal256Builder).Append(s.(*Decimal256).Value) + case arrow.STRUCT: + sb := bldr.(*array.StructBuilder) + sb.Append(true) + + v := s.(*Struct).Value + m, ok := v.(map[string]any) + if !ok { + reflectedMap := reflect.ValueOf(v) + if reflectedMap.Kind() != reflect.Map { + panic(fmt.Sprintf("expected map[string]any, got %T", v)) + } + + m = make(map[string]any, reflectedMap.Len()) + for _, key := range reflectedMap.MapKeys() { + if key.Kind() != reflect.String { + panic(fmt.Sprintf("expected map[string]any, got %T", v)) + } + value := reflectedMap.MapIndex(key) + m[key.String()] = value.Interface() + } + } + + names := make(map[string]struct{}, len(m)) + for k := range m { + names[k] = struct{}{} + } + + st := sb.Type().(*arrow.StructType) + for i, f := range st.Fields() { + sc := NewScalar(sb.FieldBuilder(i).Type()) + if sv, ok := m[f.Name]; ok { + if err := sc.Set(sv); err != nil { + panic(err) + } + delete(names, f.Name) + } + + AppendToBuilder(sb.FieldBuilder(i), sc) + } + if len(names) > 0 { + panic(fmt.Errorf("struct has extra fields: %+v", maps.Keys(names))) + } + + case arrow.LIST: + lb := bldr.(*array.ListBuilder) + lb.Append(true) + for _, v := range s.(*List).Value { + AppendToBuilder(lb.ValueBuilder(), v) + } + case arrow.EXTENSION: + switch { + case arrow.TypeEqual(s.DataType(), types.ExtensionTypes.UUID): + bldr.(*types.UUIDBuilder).Append(s.(*UUID).Value) + case arrow.TypeEqual(s.DataType(), types.ExtensionTypes.JSON): + bldr.(*types.JSONBuilder).AppendBytes(s.(*JSON).Value) + case arrow.TypeEqual(s.DataType(), types.ExtensionTypes.MAC): + bldr.(*types.MACBuilder).Append(s.(*Mac).Value) + case arrow.TypeEqual(s.DataType(), types.ExtensionTypes.Inet): + bldr.(*types.InetBuilder).Append(s.(*Inet).Value) + default: + panic("not implemented extension: " + s.DataType().Name()) + } + default: + panic("not implemented: " + s.DataType().String()) + } +} + +func AppendToRecordBuilder(bldr *array.RecordBuilder, vector Vector) { + for i, scalar := range vector { + AppendToBuilder(bldr.Field(i), scalar) + } +} diff --git a/plugin-sdk/scalar/scalar_test.go b/plugin-sdk/scalar/scalar_test.go new file mode 100644 index 00000000..16be3b9e --- /dev/null +++ b/plugin-sdk/scalar/scalar_test.go @@ -0,0 +1,158 @@ +package scalar + +import ( + "testing" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/stretchr/testify/assert" +) + +func TestNewScalar(t *testing.T) { + tl := []struct { + dt arrow.DataType + input any + }{ + {dt: arrow.PrimitiveTypes.Uint8}, + {dt: arrow.PrimitiveTypes.Uint16}, + {dt: arrow.PrimitiveTypes.Uint32}, + {dt: arrow.PrimitiveTypes.Uint64}, + {dt: arrow.PrimitiveTypes.Int8}, + {dt: arrow.PrimitiveTypes.Int16}, + {dt: arrow.PrimitiveTypes.Int32}, + {dt: arrow.PrimitiveTypes.Int64}, + {dt: arrow.PrimitiveTypes.Float32}, + {dt: arrow.PrimitiveTypes.Float64}, + + {dt: arrow.BinaryTypes.String}, + {dt: arrow.BinaryTypes.Binary}, + {dt: arrow.BinaryTypes.LargeString}, + {dt: arrow.BinaryTypes.LargeBinary}, + + {dt: arrow.FixedWidthTypes.Boolean}, + {dt: arrow.FixedWidthTypes.Date32, input: "2006-01-02"}, + {dt: arrow.FixedWidthTypes.Date64, input: "2006-01-02"}, + {dt: arrow.FixedWidthTypes.Time32s, input: "21:14:00"}, + {dt: arrow.FixedWidthTypes.Time32ms, input: "21:14:00.709"}, + {dt: arrow.FixedWidthTypes.Time64us, input: "21:14:00.709229"}, + {dt: arrow.FixedWidthTypes.Time64ns, input: "21:14:00.709227000"}, + {dt: arrow.FixedWidthTypes.Timestamp_ns, input: "2006-01-02 15:04:05.999999999"}, + {dt: arrow.FixedWidthTypes.Timestamp_us, input: "2006-01-02 15:04:05.999999999"}, + {dt: arrow.FixedWidthTypes.Timestamp_ms, input: "2006-01-02 15:04:05.999999999"}, + {dt: arrow.FixedWidthTypes.Timestamp_s, input: "2006-01-02 15:04:05.999999999"}, + {dt: arrow.FixedWidthTypes.Duration_ns}, + {dt: arrow.FixedWidthTypes.Duration_ns, input: "1ns"}, + {dt: arrow.FixedWidthTypes.Duration_us}, + {dt: arrow.FixedWidthTypes.Duration_us, input: "1us"}, + {dt: arrow.FixedWidthTypes.Duration_ms}, + {dt: arrow.FixedWidthTypes.Duration_ms, input: "1ms"}, + {dt: arrow.FixedWidthTypes.Duration_s}, + {dt: arrow.FixedWidthTypes.Duration_s, input: "1s"}, + {dt: arrow.FixedWidthTypes.Float16}, + + {dt: arrow.FixedWidthTypes.DayTimeInterval, input: map[string]any{"days": 1, "milliseconds": 2}}, + {dt: arrow.FixedWidthTypes.MonthDayNanoInterval, input: map[string]any{"months": 1, "days": 2, "nanoseconds": 3}}, + {dt: arrow.FixedWidthTypes.MonthInterval, input: map[string]any{"months": 1}}, + + {dt: arrow.StructOf(arrow.Field{Name: "i64", Type: arrow.PrimitiveTypes.Int64}, arrow.Field{Name: "s", Type: arrow.BinaryTypes.String}), input: `{"i64": 1, "s": "foo"}`}, + {dt: &arrow.Decimal128Type{Precision: 10, Scale: 5}}, + {dt: &arrow.Decimal256Type{Precision: 10, Scale: 5}}, + } + + for _, tc := range tl { + tc := tc + if tc.input == nil { + tc.input = "1" + } + + t.Run("create_append:"+tc.dt.String(), func(t *testing.T) { + t.Parallel() + + bldr := array.NewBuilder(memory.DefaultAllocator, tc.dt) + defer bldr.Release() + + s := NewScalar(tc.dt) + if !arrow.TypeEqual(s.DataType(), tc.dt) { + t.Fatalf("expected %v, got %v", tc.dt, s.DataType()) + } + + assert.NoError(t, s.Set(tc.input)) + if t.Failed() { + return + } + + assert.True(t, s.IsValid()) + AppendToBuilder(bldr, s) + + t.Run("double_set_nil", genDoubleSetTest(tc.dt, tc.input, nil)) + + var str *string + t.Run("double_set_typed_nil_string", genDoubleSetTest(tc.dt, tc.input, str)) + + switch { + case + tc.dt.ID() == arrow.BOOL, + tc.dt.ID() == arrow.DATE32, + tc.dt.ID() == arrow.DATE64, + tc.dt.ID() == arrow.DURATION, + tc.dt.ID() == arrow.LARGE_STRING, + tc.dt.ID() == arrow.STRING, + tc.dt.ID() == arrow.TIME32, + tc.dt.ID() == arrow.TIME64, + tc.dt.ID() == arrow.TIMESTAMP, + arrow.IsDecimal(tc.dt.ID()), + arrow.IsNested(tc.dt.ID()): + + case arrow.IsInteger(tc.dt.ID()), arrow.IsFloating(tc.dt.ID()): + var i8 *int8 + t.Run("double_set_typed_nil_int8", genDoubleSetTest(tc.dt, tc.input, i8)) + + var i16 *int16 + t.Run("double_set_typed_nil_int16", genDoubleSetTest(tc.dt, tc.input, i16)) + + var i32 *int32 + t.Run("double_set_typed_nil_int32", genDoubleSetTest(tc.dt, tc.input, i32)) + + var i64 *int64 + t.Run("double_set_typed_nil_int64", genDoubleSetTest(tc.dt, tc.input, i64)) + + var f32 *float32 + t.Run("double_set_typed_nil_f32", genDoubleSetTest(tc.dt, tc.input, f32)) + + var f64 *float64 + t.Run("double_set_typed_nil_f64", genDoubleSetTest(tc.dt, tc.input, f64)) + + default: + var val []byte + t.Run("double_set_typed_nil_byteslice", genDoubleSetTest(tc.dt, tc.input, val)) + } + }) + } +} + +func genDoubleSetTest(dt arrow.DataType, input any, setToNil any) func(t *testing.T) { + return func(t *testing.T) { + t.Parallel() + + bldr := array.NewBuilder(memory.DefaultAllocator, dt) + defer bldr.Release() + + s := NewScalar(dt) + if !arrow.TypeEqual(s.DataType(), dt) { + t.Fatalf("expected %v, got %v", dt, s.DataType()) + } + + assert.NoError(t, s.Set(input)) + if t.Failed() { + return + } + + assert.NoError(t, s.Set(setToNil)) + if t.Failed() { + return + } + + assert.False(t, s.IsValid()) + } +} diff --git a/plugin-sdk/scalar/string.go b/plugin-sdk/scalar/string.go new file mode 100644 index 00000000..babdd83c --- /dev/null +++ b/plugin-sdk/scalar/string.go @@ -0,0 +1,118 @@ +package scalar + +import ( + "fmt" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" +) + +const nullValueStr = array.NullValueStr + +type String struct { + Valid bool + Value string +} + +func (s *String) IsValid() bool { + return s.Valid +} + +func (*String) DataType() arrow.DataType { + return arrow.BinaryTypes.String +} + +func (s *String) String() string { + if !s.Valid { + return nullValueStr + } + return s.Value +} + +func (s *String) Equal(rhs Scalar) bool { + if rhs == nil { + return false + } + r, ok := rhs.(*String) + if !ok { + return false + } + return s.Valid == r.Valid && s.Value == r.Value +} + +func (s *String) Get() any { + return s.Value +} + +func (s *String) Set(val any) error { + if val == nil { + s.Valid = false + return nil + } + + if sc, ok := val.(Scalar); ok { + if !sc.IsValid() { + s.Valid = false + return nil + } + return s.Set(sc.Get()) + } + + switch value := val.(type) { + case []byte: + s.Value = string(value) + case string: + s.Value = (value) + case fmt.Stringer: + s.Value = value.String() + case *string: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + default: + if originalSrc, ok := underlyingStringType(value); ok { + return s.Set(originalSrc) + } + return &ValidationError{Type: arrow.BinaryTypes.String, Msg: noConversion, Value: value} + } + + s.Valid = true + return nil +} + +type LargeString struct { + s String +} + +func (s *LargeString) IsValid() bool { + return s.s.Valid +} + +func (*LargeString) DataType() arrow.DataType { + return arrow.BinaryTypes.LargeString +} + +func (s *LargeString) String() string { + return s.s.String() +} + +func (s *LargeString) Equal(rhs Scalar) bool { + if rhs == nil { + return false + } + r, ok := rhs.(*LargeString) + if !ok { + return false + } + return s.s.Valid == r.s.Valid && s.s.Value == r.s.Value +} + +func (s *LargeString) Get() any { + return s.s.Get() +} + +func (s *LargeString) Set(val any) error { + return s.s.Set(val) +} diff --git a/plugin-sdk/scalar/string_test.go b/plugin-sdk/scalar/string_test.go new file mode 100644 index 00000000..97f7444d --- /dev/null +++ b/plugin-sdk/scalar/string_test.go @@ -0,0 +1,26 @@ +package scalar + +import "testing" + +func TestStringSet(t *testing.T) { + successfulTests := []struct { + source any + result String + }{ + {source: "foo", result: String{Value: "foo", Valid: true}}, + {source: _string("bar"), result: String{Value: "bar", Valid: true}}, + {source: (*string)(nil), result: String{}}, + } + + for i, tt := range successfulTests { + var d String + err := d.Set(tt.source) + if err != nil { + t.Errorf("%d: %v", i, err) + } + + if d != tt.result { + t.Errorf("%d: expected %v to convert to %v, but it was %v", i, tt.source, tt.result, d) + } + } +} diff --git a/plugin-sdk/scalar/struct.go b/plugin-sdk/scalar/struct.go new file mode 100644 index 00000000..088466da --- /dev/null +++ b/plugin-sdk/scalar/struct.go @@ -0,0 +1,118 @@ +package scalar + +import ( + "encoding/base64" + "encoding/json" + "reflect" + + "github.com/apache/arrow/go/v16/arrow" +) + +type Struct struct { + Valid bool + Value any + + Type *arrow.StructType +} + +func (s *Struct) IsValid() bool { + return s.Valid +} + +func (s *Struct) Equal(rhs Scalar) bool { + if rhs == nil { + return false + } + r, ok := rhs.(*Struct) + if !ok { + return false + } + return s.Valid == r.Valid && arrow.TypeEqual(s.Type, r.Type) && reflect.DeepEqual(s.Value, r.Value) +} + +func (s *Struct) String() string { + if !s.Valid { + return nullValueStr + } + b, _ := json.Marshal(s.Value) + return string(b) +} + +func (s *Struct) Get() any { + return s.Value +} + +func (s *Struct) Set(val any) error { + if val == nil { + s.Valid = false + return nil + } + + if sc, ok := val.(Scalar); ok { + if !sc.IsValid() { + s.Valid = false + return nil + } + return s.Set(sc.Get()) + } + + switch value := val.(type) { + case string: + var x map[string]any + if err := json.Unmarshal([]byte(value), &x); err != nil { + return err + } + for name := range x { + if f, ok := s.Type.FieldByName(name); ok { + xs, ok := x[name].(string) + if !ok { + continue + } + switch { + case arrow.TypeEqual(f.Type, arrow.BinaryTypes.Binary): + v, err := base64.StdEncoding.DecodeString(xs) + if err != nil { + return err + } + x[name] = v + case arrow.TypeEqual(f.Type, arrow.BinaryTypes.LargeBinary): + v, err := base64.StdEncoding.DecodeString(xs) + if err != nil { + return err + } + x[name] = v + } + } + } + s.Value = x + + case []byte: + var x map[string]any + if err := json.Unmarshal(value, &x); err != nil { + return err + } + s.Value = x + + case *string: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + + default: + s.Value = val + } + + if rv := reflect.ValueOf(val); rv.Kind() == reflect.Pointer && !rv.Elem().IsValid() { // typed nil + s.Valid = false + return nil + } + + s.Valid = true + return nil +} + +func (s *Struct) DataType() arrow.DataType { + return s.Type +} diff --git a/plugin-sdk/scalar/struct_test.go b/plugin-sdk/scalar/struct_test.go new file mode 100644 index 00000000..25e1b9da --- /dev/null +++ b/plugin-sdk/scalar/struct_test.go @@ -0,0 +1,147 @@ +package scalar + +import ( + "encoding/json" + "strconv" + "testing" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" +) + +func TestStructEncodeDecode(t *testing.T) { + tl := []struct { + name string + dt *arrow.StructType + input any + }{ + {name: "binary", dt: arrow.StructOf(arrow.Field{Name: "binary", Type: arrow.BinaryTypes.Binary}), input: `{"binary":"7049Ug=="}`}, + {name: "uuid", dt: arrow.StructOf(arrow.Field{Name: "uuid", Type: types.ExtensionTypes.UUID}), input: `{"uuid":"f81d4fae-7dec-11d0-a765-00a0c91e6bf6"}`}, + } + + for _, tc := range tl { + t.Run(tc.name, func(t *testing.T) { + bldr := array.NewBuilder(memory.DefaultAllocator, tc.dt) + defer bldr.Release() + + s := NewScalar(tc.dt) + if !arrow.TypeEqual(s.DataType(), tc.dt) { + t.Fatalf("expected %v, got %v", tc.dt, s.DataType()) + } + + assert.NoError(t, s.Set(tc.input)) + if t.Failed() { + return + } + + assert.True(t, s.IsValid()) + AppendToBuilder(bldr, s) + arr := bldr.NewArray() + one := arr.GetOneForMarshal(0) + json, err := json.Marshal(one) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, tc.input, string(json)) + }) + } +} + +func TestStructMissingKeys(t *testing.T) { + tl := []struct { + dt *arrow.StructType + input any + expectPanic bool + }{ + {dt: arrow.StructOf(arrow.Field{Name: "i64", Type: arrow.PrimitiveTypes.Int64}, arrow.Field{Name: "s", Type: arrow.BinaryTypes.String}), input: `{"i64": 1, "s": "foo"}`, expectPanic: false}, + {dt: arrow.StructOf(arrow.Field{Name: "i64", Type: arrow.PrimitiveTypes.Int64}, arrow.Field{Name: "s", Type: arrow.BinaryTypes.String}), input: `{"i64": 1, "s": "foo", "extra":"bar"}`, expectPanic: true}, + } + + for idx, tc := range tl { + tc := tc + t.Run(strconv.Itoa(idx), func(t *testing.T) { + panicked := false + defer func() { + if t.Failed() { + return + } + + if panicked && !tc.expectPanic { + t.Errorf("unexpected panic") + } + if !panicked && tc.expectPanic { + t.Errorf("expected panic") + } + }() + + defer func() { + if r := recover(); r != nil { + panicked = true + } + }() + + bldr := array.NewBuilder(memory.DefaultAllocator, tc.dt) + defer bldr.Release() + + s := NewScalar(tc.dt) + if !arrow.TypeEqual(s.DataType(), tc.dt) { + t.Fatalf("expected %v, got %v", tc.dt, s.DataType()) + } + + assert.NoError(t, s.Set(tc.input)) + if t.Failed() { + return + } + + assert.True(t, s.IsValid()) + AppendToBuilder(bldr, s) + }) + } +} + +func TestStructSet(t *testing.T) { + type testType any + + tl := []struct { + schema arrow.DataType + input any + }{ + { + schema: arrow.StructOf(arrow.Field{Name: "a", Type: arrow.PrimitiveTypes.Int64}, arrow.Field{Name: "b", Type: arrow.PrimitiveTypes.Int64}), + input: map[string]any{"a": 1, "b": 2}, + }, + { + schema: arrow.StructOf(arrow.Field{Name: "a", Type: arrow.PrimitiveTypes.Int64}, arrow.Field{Name: "b", Type: arrow.PrimitiveTypes.Int64}), + input: map[string]testType{"a": 1, "b": 2}, + }, + { + schema: arrow.StructOf(arrow.Field{Name: "nested", Type: arrow.StructOf(arrow.Field{Name: "a", Type: arrow.PrimitiveTypes.Int64}, arrow.Field{Name: "b", Type: arrow.PrimitiveTypes.Int64})}), + input: map[string]any{"x": map[string]any{"a": 1, "b": 2}}, + }, + { + schema: arrow.StructOf(arrow.Field{Name: "nested", Type: arrow.StructOf(arrow.Field{Name: "a", Type: arrow.PrimitiveTypes.Int64}, arrow.Field{Name: "b", Type: arrow.PrimitiveTypes.Int64})}), + input: map[string]testType{"x": map[string]testType{"a": 1, "b": 2}}, + }, + { + schema: arrow.StructOf(arrow.Field{Name: "nested", Type: arrow.StructOf(arrow.Field{Name: "a", Type: arrow.PrimitiveTypes.Int64}, arrow.Field{Name: "b", Type: arrow.PrimitiveTypes.Int64})}), + input: map[string]testType{"x": map[string]any{"a": 1, "b": 2}}, + }, + { + schema: arrow.StructOf(arrow.Field{Name: "nested", Type: arrow.StructOf(arrow.Field{Name: "a", Type: arrow.PrimitiveTypes.Int64}, arrow.Field{Name: "b", Type: arrow.PrimitiveTypes.Int64})}), + input: map[string]any{"x": map[string]testType{"a": 1, "b": 2}}, + }, + } + + for idx, tc := range tl { + tc := tc + t.Run(strconv.Itoa(idx), func(t *testing.T) { + s := NewScalar(tc.schema) + require.NoError(t, s.Set(tc.input)) + }) + } +} diff --git a/plugin-sdk/scalar/time.go b/plugin-sdk/scalar/time.go new file mode 100644 index 00000000..f060a32b --- /dev/null +++ b/plugin-sdk/scalar/time.go @@ -0,0 +1,96 @@ +package scalar + +import ( + "github.com/apache/arrow/go/v16/arrow" +) + +type Time struct { + Int + Unit arrow.TimeUnit +} + +func (s *Time) DataType() arrow.DataType { + switch s.getBitWidth() { + case 64: + return &arrow.Time64Type{Unit: s.Unit} + case 32: + return &arrow.Time32Type{Unit: s.Unit} + default: + panic("unsupported bit width") + } +} + +func (s *Time) Equal(rhs Scalar) bool { + if rhs == nil { + return false + } + r, ok := rhs.(*Time) + if !ok { + return false + } + return s.Valid == r.Valid && s.Unit == r.Unit && s.getBitWidth() == r.getBitWidth() && s.Value == r.Value +} + +func (s *Time) String() string { + if !s.Valid { + return nullValueStr + } + + switch s.getBitWidth() { + case 64: + return arrow.Time64(s.Int.Value).FormattedString(s.Unit) + case 32: + return arrow.Time32(s.Int.Value).FormattedString(s.Unit) + default: + panic("unsupported bit width") + } +} + +func (s *Time) Get() any { + if !s.Valid { + return nil + } + switch s.getBitWidth() { + case 64: + return arrow.Time64(s.Int.Get().(int64)) + case 32: + return arrow.Time32(s.Int.Get().(int64)) + default: + panic("unknown bit width") + } +} + +func (s *Time) Set(value any) error { + switch v := value.(type) { + case arrow.Time32: + return s.Int.Set(int64(v)) + case arrow.Time64: + return s.Int.Set(int64(v)) + + case string: + switch s.getBitWidth() { + case 64: + t64, err := arrow.Time64FromString(v, s.Unit) + if err != nil { + return err + } + return s.Set(t64) + case 32: + t32, err := arrow.Time32FromString(v, s.Unit) + if err != nil { + return err + } + return s.Set(t32) + default: + return s.Int.Set(v) + } + case *string: + if v == nil { + s.Valid = false + return nil + } + return s.Set(*v) + default: + return s.Int.Set(value) + } +} diff --git a/plugin-sdk/scalar/timestamp.go b/plugin-sdk/scalar/timestamp.go new file mode 100644 index 00000000..fbc412f3 --- /dev/null +++ b/plugin-sdk/scalar/timestamp.go @@ -0,0 +1,151 @@ +package scalar + +import ( + "encoding" + "fmt" + "math" + "time" + + "github.com/apache/arrow/go/v16/arrow" +) + +const ( + // this is the default format used by time.Time.String() + defaultStringFormat = "2006-01-02 15:04:05.999999999 -0700 MST" + + // these are used by Arrow string format (time is in UTC) + arrowStringFormat = "2006-01-02 15:04:05.999999999" + arrowStringFormatNew = "2006-01-02 15:04:05.999999999Z" +) + +type Timestamp struct { + Valid bool + Value time.Time + Type *arrow.TimestampType +} + +func (s *Timestamp) IsValid() bool { + return s.Valid +} + +func (s *Timestamp) DataType() arrow.DataType { + return s.Type +} + +func (s *Timestamp) Equal(rhs Scalar) bool { + if rhs == nil { + return false + } + r, ok := rhs.(*Timestamp) + if !ok { + return false + } + return s.Valid == r.Valid && s.Value.Equal(r.Value) +} + +func (s *Timestamp) String() string { + if !s.Valid { + return nullValueStr + } + return s.Value.Format(time.RFC3339) +} + +func (s *Timestamp) Get() any { + return s.Value +} + +func (s *Timestamp) Set(val any) error { + if val == nil { + s.Valid = false + return nil + } + + if sc, ok := val.(Scalar); ok { + if !sc.IsValid() { + s.Valid = false + return nil + } + return s.Set(sc.Get()) + } + + switch value := val.(type) { + case int: + s.Value = time.Unix(int64(value), 0).UTC() + case int64: + s.Value = time.Unix(value, 0).UTC() + case uint64: + if value > math.MaxInt64 { + return &ValidationError{Type: s.DataType(), Msg: "uint64 greater than MaxInt64", Value: value} + } + s.Value = time.Unix(int64(value), 0).UTC() + case time.Time: + s.Value = value.UTC() + case *time.Time: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case string: + return s.DecodeText([]byte(value)) + case *string: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + default: + if originalSrc, ok := underlyingTimeType(val); ok { + return s.Set(originalSrc) + } + if value, ok := value.(encoding.TextMarshaler); ok { + text, err := value.MarshalText() + if err == nil { + return s.Set(string(text)) + } + // fall through to String() method + } + if value, ok := value.(fmt.Stringer); ok { + str := value.String() + return s.Set(str) + } + return &ValidationError{Type: s.DataType(), Msg: noConversion, Value: value} + } + s.Valid = true + return nil +} + +func (s *Timestamp) DecodeText(src []byte) error { + if len(src) == 0 { + s.Valid = false + return nil + } + + sbuf := string(src) + // nolint:gocritic,revive + switch sbuf { + default: + var tim time.Time + var err error + + if len(sbuf) > len(defaultStringFormat)+1 && sbuf[len(defaultStringFormat)+1] == 'm' { + sbuf = sbuf[:len(defaultStringFormat)] + } + + // there is no good way of detecting format, so we just try few of them + for _, format := range []string{ + time.RFC3339, + defaultStringFormat, + arrowStringFormat, + arrowStringFormatNew, + } { + tim, err = time.Parse(format, sbuf) + if err == nil { + s.Value = tim.UTC() + s.Valid = true + return nil + } + } + return &ValidationError{Type: s.DataType(), Msg: "cannot parse timestamp", Value: sbuf, Err: err} + } +} diff --git a/plugin-sdk/scalar/timestamp_test.go b/plugin-sdk/scalar/timestamp_test.go new file mode 100644 index 00000000..130de4f7 --- /dev/null +++ b/plugin-sdk/scalar/timestamp_test.go @@ -0,0 +1,143 @@ +package scalar + +import ( + "strconv" + "testing" + "time" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type TimestampSt struct { + time.Time +} + +func TestTimestampSet(t *testing.T) { + type _time time.Time + + timeInstance := time.Date(2105, 7, 23, 22, 23, 37, 750076110, time.UTC) + timeRFC3339NanoBytes, _ := timeInstance.MarshalText() + + successfulTests := []struct { + source any + result Timestamp + }{ + {source: time.Date(1900, 1, 1, 0, 0, 0, 0, time.Local), result: Timestamp{Value: time.Date(1900, 1, 1, 0, 0, 0, 0, time.Local), Valid: true}}, + {source: time.Date(1970, 1, 1, 0, 0, 0, 0, time.Local), result: Timestamp{Value: time.Date(1970, 1, 1, 0, 0, 0, 0, time.Local), Valid: true}}, + {source: time.Date(1999, 12, 31, 12, 59, 59, 0, time.Local), result: Timestamp{Value: time.Date(1999, 12, 31, 12, 59, 59, 0, time.Local), Valid: true}}, + {source: time.Date(2000, 1, 1, 0, 0, 0, 0, time.Local), result: Timestamp{Value: time.Date(2000, 1, 1, 0, 0, 0, 0, time.Local), Valid: true}}, + {source: time.Date(2000, 1, 1, 0, 0, 1, 0, time.Local), result: Timestamp{Value: time.Date(2000, 1, 1, 0, 0, 1, 0, time.Local), Valid: true}}, + {source: time.Date(2200, 1, 1, 0, 0, 0, 0, time.Local), result: Timestamp{Value: time.Date(2200, 1, 1, 0, 0, 0, 0, time.Local), Valid: true}}, + {source: int(time.Date(2000, 1, 1, 0, 0, 0, 0, time.Local).Unix()), result: Timestamp{Value: time.Date(2000, 1, 1, 0, 0, 0, 0, time.Local), Valid: true}}, + {source: uint64(time.Date(2000, 1, 1, 0, 0, 0, 0, time.Local).Unix()), result: Timestamp{Value: time.Date(2000, 1, 1, 0, 0, 0, 0, time.Local), Valid: true}}, + {source: time.Date(2000, 1, 1, 0, 0, 0, 0, time.Local).Unix(), result: Timestamp{Value: time.Date(2000, 1, 1, 0, 0, 0, 0, time.Local), Valid: true}}, + {source: _time(time.Date(1970, 1, 1, 0, 0, 0, 0, time.Local)), result: Timestamp{Value: time.Date(1970, 1, 1, 0, 0, 0, 0, time.Local), Valid: true}}, + {source: string(timeRFC3339NanoBytes), result: Timestamp{Value: time.Date(2105, 7, 23, 22, 23, 37, 750076110, time.UTC), Valid: true}}, + {source: "2150-10-15 07:25:09.75007611 +0000 UTC", result: Timestamp{Value: time.Date(2150, 10, 15, 7, 25, 9, 750076110, time.UTC), Valid: true}}, + {source: timeInstance.String(), result: Timestamp{Value: time.Date(2105, 7, 23, 22, 23, 37, 750076110, time.UTC), Valid: true}}, + {source: TimestampSt{timeInstance}, result: Timestamp{Value: time.Date(2105, 7, 23, 22, 23, 37, 750076110, time.UTC), Valid: true}}, + {source: "", result: Timestamp{}}, + {source: &Timestamp{Value: time.Date(2105, 7, 23, 22, 23, 37, 750076110, time.UTC), Valid: true}, result: Timestamp{Value: time.Date(2105, 7, 23, 22, 23, 37, 750076110, time.UTC), Valid: true}}, + } + + for i, tt := range successfulTests { + var r Timestamp + err := r.Set(tt.source) + if err != nil { + t.Errorf("%d: %v", i, err) + continue + } + + if !r.Equal(&tt.result) { + t.Errorf("%d: %v != %v", i, r, tt.result) + } + } +} + +func TestTimestampDoubleSet(t *testing.T) { + var r Timestamp + assert.NoError(t, r.Set("2105-07-23 22:23:37.75007611 +0000 UTC")) + + r2 := r + assert.NoError(t, r.Set("")) + if r.Equal(&r2) { + t.Errorf("%v = %v, expected null", r, r2) + } +} + +func TestAppendToBuilderTimestamp(t *testing.T) { + for idx, tc := range []struct { + Unit arrow.TimeUnit + Input string + Expected string + }{ + // Input format: arrowStringFormat + { + Unit: arrow.Second, + Input: "1999-01-08 04:05:06.123456789", + Expected: "1999-01-08 04:05:06Z", + }, + { + Unit: arrow.Millisecond, + Input: "1999-01-08 04:05:06.123456789", + Expected: "1999-01-08 04:05:06.123Z", + }, + { + Unit: arrow.Microsecond, + Input: "1999-01-08 04:05:06.123456789", + Expected: "1999-01-08 04:05:06.123456Z", + }, + { + Unit: arrow.Nanosecond, + Input: "1999-01-08 04:05:06.123456789", + Expected: "1999-01-08 04:05:06.123456789Z", + }, + // Input format: arrowStringFormatNew + { + Unit: arrow.Second, + Input: "1999-01-08 04:05:06.123456789Z", + Expected: "1999-01-08 04:05:06Z", + }, + { + Unit: arrow.Millisecond, + Input: "1999-01-08 04:05:06.123456789Z", + Expected: "1999-01-08 04:05:06.123Z", + }, + { + Unit: arrow.Microsecond, + Input: "1999-01-08 04:05:06.123456789Z", + Expected: "1999-01-08 04:05:06.123456Z", + }, + { + Unit: arrow.Nanosecond, + Input: "1999-01-08 04:05:06.123456789Z", + Expected: "1999-01-08 04:05:06.123456789Z", + }, + } { + tc := tc + t.Run(strconv.FormatInt(int64(idx), 10), func(t *testing.T) { + timestamp := Timestamp{ + Type: &arrow.TimestampType{ + Unit: tc.Unit, + TimeZone: "UTC", + }, + } + err := timestamp.Set(tc.Input) + if err != nil { + t.Fatal(err) + } + + bldr := array.NewTimestampBuilder(memory.DefaultAllocator, timestamp.Type) + AppendToBuilder(bldr, ×tamp) + + arr := bldr.NewArray().(*array.Timestamp) + actual := arr.ValueStr(0) + + require.Equal(t, tc.Expected, actual) + }) + } +} diff --git a/plugin-sdk/scalar/type_test.go b/plugin-sdk/scalar/type_test.go new file mode 100644 index 00000000..bced52ee --- /dev/null +++ b/plugin-sdk/scalar/type_test.go @@ -0,0 +1,10 @@ +package scalar + +// Test for renamed types +type _string string + +type _bool bool + +type _int8 int8 + +type _byteSlice []byte diff --git a/plugin-sdk/scalar/uint.go b/plugin-sdk/scalar/uint.go new file mode 100644 index 00000000..e5a6bc74 --- /dev/null +++ b/plugin-sdk/scalar/uint.go @@ -0,0 +1,248 @@ +package scalar + +import ( + "math" + "strconv" + + "github.com/apache/arrow/go/v16/arrow" +) + +type Uint struct { + Valid bool + Value uint64 + BitWidth uint8 // defaults to 64 +} + +func (s *Uint) IsValid() bool { + return s.Valid +} + +func (s *Uint) DataType() arrow.DataType { + switch s.getBitWidth() { + case 64: + return arrow.PrimitiveTypes.Uint64 + case 32: + return arrow.PrimitiveTypes.Uint32 + case 16: + return arrow.PrimitiveTypes.Uint16 + case 8: + return arrow.PrimitiveTypes.Uint8 + default: + panic("invalid bit width") + } +} + +func (s *Uint) String() string { + if !s.Valid { + return nullValueStr + } + return strconv.FormatUint(s.Value, 10) +} + +func (s *Uint) Equal(rhs Scalar) bool { + if rhs == nil { + return false + } + r, ok := rhs.(*Uint) + if !ok { + return false + } + return s.getBitWidth() == r.getBitWidth() && s.Valid == r.Valid && s.Value == r.Value +} + +func (s *Uint) Get() any { + return s.Value +} + +func (s *Uint) Set(val any) error { + if val == nil { + s.Valid = false + return nil + } + + if sc, ok := val.(Scalar); ok { + if !sc.IsValid() { + s.Valid = false + return nil + } + return s.Set(sc.Get()) + } + + switch value := val.(type) { + case int8: + if value < 0 { + return &ValidationError{Type: s.DataType(), Msg: "int8 less than 0", Value: value} + } + return s.Set(uint64(value)) + case int16: + if value < 0 { + return &ValidationError{Type: s.DataType(), Msg: "int16 less than 0", Value: value} + } + return s.Set(uint64(value)) + case int32: + if value < 0 { + return &ValidationError{Type: s.DataType(), Msg: "int32 less than 0", Value: value} + } + return s.Set(uint64(value)) + case int64: + if value < 0 { + return &ValidationError{Type: s.DataType(), Msg: "int64 less than 0", Value: value} + } + return s.Set(uint64(value)) + case int: + if value < 0 { + return &ValidationError{Type: s.DataType(), Msg: "int less than 0", Value: value} + } + return s.Set(uint64(value)) + case uint8: + return s.Set(uint64(value)) + case uint16: + return s.Set(uint64(value)) + case uint32: + return s.Set(uint64(value)) + case uint64: + if err := s.validateValue(value); err != nil { + return err + } + s.Value = value + case uint: + return s.Set(uint64(value)) + case float32: + switch { + case value < 0: + return &ValidationError{Type: s.DataType(), Msg: "float32 less than 0", Value: value} + case s.getBitWidth() == 32 && value > math.MaxUint32: + return &ValidationError{Type: s.DataType(), Msg: "float32 is greater than MaxUint32", Value: value} + case value > math.MaxUint64: + return &ValidationError{Type: s.DataType(), Msg: "float32 is greater than MaxUint64", Value: value} + } + + return s.Set(uint64(value)) + case float64: + switch { + case value < 0: + return &ValidationError{Type: s.DataType(), Msg: "float64 less than 0", Value: value} + case s.getBitWidth() == 32 && value > math.MaxUint32: + return &ValidationError{Type: s.DataType(), Msg: "float64 is greater than MaxUint32", Value: value} + case value > math.MaxUint64: + return &ValidationError{Type: s.DataType(), Msg: "float64 is greater than MaxUint64", Value: value} + } + return s.Set(uint64(value)) + case string: + v, err := strconv.ParseUint(value, 10, 64) + if err != nil { + return &ValidationError{Type: s.DataType(), Msg: "invalid string", Value: value} + } + return s.Set(v) + case *string: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *int8: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *int16: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *int32: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *int64: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *int: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *uint8: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *uint16: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *uint32: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *uint64: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *uint: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *float32: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case *float64: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + default: + if originalSrc, ok := underlyingNumberType(value); ok { + return s.Set(originalSrc) + } + return &ValidationError{Type: s.DataType(), Msg: noConversion, Value: value} + } + s.Valid = true + return nil +} + +func (s *Uint) validateValue(value uint64) error { + switch s.getBitWidth() { + case 8: + if value > math.MaxUint8 { + return &ValidationError{Type: s.DataType(), Msg: "value greater than MaxUint8", Value: value} + } + case 16: + if value > math.MaxUint16 { + return &ValidationError{Type: s.DataType(), Msg: "value greater than MaxUint16", Value: value} + } + case 32: + if value > math.MaxUint32 { + return &ValidationError{Type: s.DataType(), Msg: "value greater than MaxUint32", Value: value} + } + } + return nil +} + +func (s *Uint) getBitWidth() uint8 { + if s.BitWidth == 0 { + return 64 // default + } + return s.BitWidth +} diff --git a/plugin-sdk/scalar/uint_test.go b/plugin-sdk/scalar/uint_test.go new file mode 100644 index 00000000..d9c53424 --- /dev/null +++ b/plugin-sdk/scalar/uint_test.go @@ -0,0 +1,96 @@ +package scalar + +import ( + "math" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestUint64Set(t *testing.T) { + successfulTests := []struct { + source any + expect Uint + }{ + {source: int8(1), expect: Uint{Value: 1, Valid: true}}, + {source: int16(1), expect: Uint{Value: 1, Valid: true}}, + {source: int32(1), expect: Uint{Value: 1, Valid: true}}, + {source: int64(1), expect: Uint{Value: 1, Valid: true}}, + {source: uint8(1), expect: Uint{Value: 1, Valid: true}}, + {source: uint16(1), expect: Uint{Value: 1, Valid: true}}, + {source: uint32(1), expect: Uint{Value: 1, Valid: true}}, + {source: uint64(1), expect: Uint{Value: 1, Valid: true}}, + {source: float32(1), expect: Uint{Value: 1, Valid: true}}, + {source: float64(1), expect: Uint{Value: 1, Valid: true}}, + {source: "1", expect: Uint{Value: 1, Valid: true}}, + {source: _int8(1), expect: Uint{Value: 1, Valid: true}}, + {source: &Uint{Value: 1, Valid: true}, expect: Uint{Value: 1, Valid: true}}, + } + + for _, bitWidth := range []uint8{8, 16, 32, 64} { + bitWidth := bitWidth + t.Run(strconv.Itoa(int(bitWidth)), func(t *testing.T) { + t.Parallel() + + for i, tt := range successfulTests { + r := Uint{BitWidth: bitWidth} + err := r.Set(tt.source) + if err != nil { + t.Errorf("%d: %v", i, err) + } + + tt.expect.BitWidth = bitWidth + if !r.Equal(&tt.expect) { + t.Errorf("%d: %v != %v", i, r, tt.expect) + } + } + }) + } +} + +func TestUintOverflows(t *testing.T) { + cases := []struct { + source any + bitWidth uint8 + expectError bool + }{ + {source: int16(math.MaxUint8), bitWidth: 8, expectError: false}, + {source: int16(math.MaxUint8 + 1), bitWidth: 8, expectError: true}, + {source: uint16(math.MaxUint8), bitWidth: 8, expectError: false}, + {source: float32(math.MaxUint8), bitWidth: 8, expectError: false}, + {source: float32(math.MaxUint8 + 1), bitWidth: 8, expectError: true}, + {source: &Uint{Value: math.MaxUint8 + 1, Valid: true}, bitWidth: 8, expectError: true}, + + {source: int32(math.MaxUint16), bitWidth: 16, expectError: false}, + {source: int32(math.MaxUint16 + 1), bitWidth: 16, expectError: true}, + {source: uint16(math.MaxUint16), bitWidth: 16, expectError: false}, + {source: float32(math.MaxUint16), bitWidth: 16, expectError: false}, + {source: float32(math.MaxUint16 + 1), bitWidth: 16, expectError: true}, + + {source: int64(math.MaxUint32), bitWidth: 32, expectError: false}, + {source: int64(math.MaxUint32 + 1), bitWidth: 32, expectError: true}, + {source: uint32(math.MaxUint32), bitWidth: 32, expectError: false}, + {source: uint64(math.MaxUint32), bitWidth: 32, expectError: false}, + {source: uint64(math.MaxUint32 + 1), bitWidth: 32, expectError: true}, + {source: float64(math.MaxUint32), bitWidth: 32, expectError: false}, + {source: float64(math.MaxUint32 + 1), bitWidth: 32, expectError: true}, + + {source: float32(math.MaxFloat32), bitWidth: 64, expectError: true}, + {source: float64(math.MaxFloat64), bitWidth: 64, expectError: true}, + } + for idx, tc := range cases { + tc := tc + t.Run(strconv.Itoa(idx), func(t *testing.T) { + t.Parallel() + + r := Uint{BitWidth: tc.bitWidth} + err := r.Set(tc.source) + if tc.expectError { + assert.Errorf(t, err, "with %T %#v", tc.source, tc.source) + } else { + assert.NoErrorf(t, err, "with %T %#v", tc.source, tc.source) + } + }) + } +} diff --git a/plugin-sdk/scalar/uuid.go b/plugin-sdk/scalar/uuid.go new file mode 100644 index 00000000..c55d0f43 --- /dev/null +++ b/plugin-sdk/scalar/uuid.go @@ -0,0 +1,123 @@ +package scalar + +import ( + "encoding/hex" + "fmt" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/google/uuid" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" +) + +type UUID struct { + Valid bool + Value uuid.UUID +} + +func (s *UUID) IsValid() bool { + return s.Valid +} + +func (*UUID) DataType() arrow.DataType { + return types.ExtensionTypes.UUID +} + +func (s *UUID) String() string { + if !s.Valid { + return nullValueStr + } + return s.Value.String() +} + +func (s *UUID) Equal(rhs Scalar) bool { + if rhs == nil { + return false + } + r, ok := rhs.(*UUID) + if !ok { + return false + } + return s.Valid == r.Valid && s.Value == r.Value +} + +func (s *UUID) Get() any { + return s.Value +} + +func (s *UUID) Set(src any) error { + if src == nil { + return nil + } + + if sc, ok := src.(Scalar); ok { + if !sc.IsValid() { + s.Valid = false + return nil + } + return s.Set(sc.Get()) + } + + switch value := src.(type) { + case fmt.Stringer: + value2 := value.String() + return s.Set(value2) + case [16]byte: + s.Value = uuid.UUID(value) + case *[]byte: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + case []byte: + if value == nil { + s.Valid = false + return nil + } + if len(value) != 16 { + return &ValidationError{Type: types.ExtensionTypes.UUID, Msg: "[]byte must be 16 bytes to convert to UUID", Value: value} + } + copy(s.Value[:], value) + case string: + uuidVal, err := parseUUID(value) + if err != nil { + return err + } + s.Value = uuidVal + case *string: + if value == nil { + s.Valid = false + return nil + } + return s.Set(*value) + default: + if originalSrc, ok := underlyingUUIDType(src); ok { + return s.Set(originalSrc) + } + return &ValidationError{Type: types.ExtensionTypes.UUID, Msg: noConversion, Value: value} + } + s.Valid = true + return nil +} + +// parseUUID converts a string UUID in standard form to a byte array. +func parseUUID(src string) (dst [16]byte, err error) { + switch len(src) { + case 36: + src = src[0:8] + src[9:13] + src[14:18] + src[19:23] + src[24:] + case 32: + // dashes already stripped, assume valid + default: + // assume invalid. + return dst, &ValidationError{Type: types.ExtensionTypes.UUID, Msg: fmt.Sprintf("invalid %d UUID length", len(src)), Value: src} + } + + buf, err := hex.DecodeString(src) + if err != nil { + return dst, err + } + + copy(dst[:], buf) + return dst, err +} diff --git a/plugin-sdk/scalar/uuid_test.go b/plugin-sdk/scalar/uuid_test.go new file mode 100644 index 00000000..8a4fa0fa --- /dev/null +++ b/plugin-sdk/scalar/uuid_test.go @@ -0,0 +1,82 @@ +package scalar + +import ( + "testing" + + "github.com/google/uuid" +) + +type SomeUUIDWrapper struct { + SomeUUIDType +} + +type SomeUUIDType [16]byte + +type StringUUIDType string + +func (s StringUUIDType) String() string { + return string(s) +} + +func TestUUIDSet(t *testing.T) { + var nilPointerByteArray *[]byte + var nilPointerString *string + + successfulTests := []struct { + source any + result UUID + }{ + { + source: nil, + result: UUID{}, + }, + { + source: [16]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + result: UUID{Value: [16]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, Valid: true}, + }, + { + source: []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + result: UUID{Value: [16]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, Valid: true}, + }, + { + source: SomeUUIDType{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + result: UUID{Value: [16]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, Valid: true}, + }, + { + source: ([]byte)(nil), + result: UUID{}, + }, + { + source: "00010203-0405-0607-0809-0a0b0c0d0e0f", + result: UUID{Value: [16]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, Valid: true}, + }, + { + source: "000102030405060708090a0b0c0d0e0f", + result: UUID{Value: [16]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, Valid: true}, + }, + { + source: StringUUIDType("00010203-0405-0607-0809-0a0b0c0d0e0f"), + result: UUID{Value: [16]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, Valid: true}, + }, + { + source: &UUID{Value: [16]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, Valid: true}, + result: UUID{Value: [16]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, Valid: true}, + }, + {source: nilPointerByteArray, result: UUID{}}, + {source: nilPointerString, result: UUID{}}, + {source: &uuid.UUID{}, result: UUID{Value: [16]byte{0}, Valid: true}}, + {source: uuid.UUID{}, result: UUID{Value: [16]byte{0}, Valid: true}}, + } + + for i, tt := range successfulTests { + var r UUID + err := r.Set(tt.source) + if err != nil { + t.Errorf("%d: %v", i, err) + } + + if !r.Equal(&tt.result) { + t.Errorf("%d: %v != %v", i, r, tt.result) + } + } +} diff --git a/plugin-sdk/scheduler/batch.go b/plugin-sdk/scheduler/batch.go new file mode 100644 index 00000000..d1181e74 --- /dev/null +++ b/plugin-sdk/scheduler/batch.go @@ -0,0 +1,222 @@ +package scheduler + +import ( + "context" + "sync" + "time" + + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/rs/zerolog" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/scalar" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/writers" +) + +const ( + DefaultBatchMaxRows = 50 + DefaultBatchTimeout = 5 * time.Second +) + +type ( + BatchSettings struct { + MaxRows int + Timeout time.Duration + } + + BatchOption func(settings *BatchSettings) +) + +func WithoutBatching() Option { + return func(s *Scheduler) { + s.batchSettings = nil + } +} + +func WithBatchOptions(options ...BatchOption) Option { + return func(s *Scheduler) { + if s.batchSettings == nil { + s.batchSettings = new(BatchSettings) + } + for _, o := range options { + o(s.batchSettings) + } + } +} + +func WithBatchMaxRows(rows int) BatchOption { + return func(s *BatchSettings) { + s.MaxRows = rows + } +} + +func WithBatchTimeout(timeout time.Duration) BatchOption { + return func(s *BatchSettings) { + s.Timeout = timeout + } +} + +func (s *BatchSettings) getBatcher(ctx context.Context, res chan<- message.SyncMessage, logger zerolog.Logger) batcherInterface { + if s == nil || s.Timeout <= 0 || s.MaxRows <= 0 { + return &nopBatcher{res: res} + } + + return &batcher{ + done: ctx.Done(), + res: res, + maxRows: s.MaxRows, + timeout: s.Timeout, + logger: logger.With().Int("max_rows", s.MaxRows).Dur("timeout_ms", s.Timeout).Logger(), + } +} + +type batcherInterface interface { + process(res *schema.Resource) + close() +} + +type nopBatcher struct { + res chan<- message.SyncMessage +} + +func (n *nopBatcher) process(resource *schema.Resource) { + n.res <- &message.SyncInsert{Record: resource.GetValues().ToArrowRecord(resource.Table.ToArrowSchema())} +} + +func (*nopBatcher) close() {} + +var _ batcherInterface = (*nopBatcher)(nil) + +type batcher struct { + done <-chan struct{} + + res chan<- message.SyncMessage + + maxRows int + timeout time.Duration + + // using sync primitives by value here implies that batcher is to be used by pointer only + // workers is a sync.Map rather than a map + mutex pair + // because worker allocation & lookup falls into one of the sync.Map use-cases, + // namely, ever-growing cache (write once, read many times). + workers sync.Map // k = table name, v = *worker + wg sync.WaitGroup + + logger zerolog.Logger +} + +type worker struct { + ch chan *schema.Resource + flush chan chan struct{} + curRows, maxRows int + builder *array.RecordBuilder // we can reuse that + res chan<- message.SyncMessage + + // debug logging + tableName string + logger *zerolog.Logger +} + +// send must be called on len(rows) > 0 +func (w *worker) send() { + w.logger.Debug().Str("table", w.tableName).Int("rows", w.curRows).Msg("send") + w.res <- &message.SyncInsert{Record: w.builder.NewRecord()} + // we need to reserve here as NewRecord (& underlying NewArray calls) reset the memory + w.builder.Reserve(w.maxRows) + w.curRows = 0 // reset +} + +func (w *worker) work(done <-chan struct{}, timeout time.Duration) { + ticker := writers.NewTicker(timeout) + defer ticker.Stop() + tickerCh := ticker.Chan() + + for { + select { + case r, ok := <-w.ch: + if !ok { + if w.curRows > 0 { + w.send() + } + return + } + + // append to builder + scalar.AppendToRecordBuilder(w.builder, r.GetValues()) + w.curRows++ + // check if we need to flush + if w.maxRows > 0 && w.curRows == w.maxRows { + w.send() + ticker.Reset(timeout) + } + + case <-tickerCh: + if w.curRows > 0 { + w.send() + } + + case ch := <-w.flush: + if w.curRows > 0 { + w.send() + ticker.Reset(timeout) + } + close(ch) + + case <-done: + // this means the request was cancelled + return // after this NO other call will succeed + } + } +} + +func (b *batcher) process(res *schema.Resource) { + table := res.Table + // already running worker + v, loaded := b.workers.Load(table.Name) + if loaded { + v.(*worker).ch <- res + return + } + + // we alloc only ch here, as it may be needed right away + // for instance, if another goroutine will get the value allocated by us + wr := &worker{ch: make(chan *schema.Resource, 5)} // 5 is quite enough + v, loaded = b.workers.LoadOrStore(table.Name, wr) + if loaded { + // means that the worker was already in tne sync.Map, so we just discard the wr value + close(wr.ch) // for GC + v.(*worker).ch <- res // send res to the already allocated worker + return + } + + // fill in the required data + // start wr + b.wg.Add(1) + go func() { + defer b.wg.Done() + + // fill in the worker fields + wr.flush = make(chan chan struct{}) + wr.maxRows = b.maxRows + wr.builder = array.NewRecordBuilder(memory.DefaultAllocator, table.ToArrowSchema()) + wr.res = b.res + wr.builder.Reserve(b.maxRows) + wr.logger = &b.logger + wr.tableName = table.Name + + // start processing + wr.work(b.done, b.timeout) + }() + + wr.ch <- res +} + +func (b *batcher) close() { + b.workers.Range(func(_, v any) bool { + close(v.(*worker).ch) + return true + }) + b.wg.Wait() +} diff --git a/plugin-sdk/scheduler/benchmark_test.go b/plugin-sdk/scheduler/benchmark_test.go new file mode 100644 index 00000000..6990da0f --- /dev/null +++ b/plugin-sdk/scheduler/benchmark_test.go @@ -0,0 +1 @@ +package scheduler diff --git a/plugin-sdk/scheduler/benchmark_test.go.backup b/plugin-sdk/scheduler/benchmark_test.go.backup new file mode 100644 index 00000000..f3d3b311 --- /dev/null +++ b/plugin-sdk/scheduler/benchmark_test.go.backup @@ -0,0 +1,429 @@ +package plugin + +import ( + "context" + "fmt" + "math/rand" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/cloudquery/plugin-pb-go/specs" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/rs/zerolog" + "golang.org/x/sync/errgroup" +) + +type BenchmarkScenario struct { + Client Client + Scheduler specs.Scheduler + Clients int + Tables int + ChildrenPerTable int + Columns int + ColumnResolvers int // number of columns with custom resolvers + ResourcesPerTable int + ResourcesPerPage int + NoPreResourceResolver bool + Concurrency uint64 +} + +func (s *BenchmarkScenario) SetDefaults() { + if s.Clients == 0 { + s.Clients = 1 + } + if s.Tables == 0 { + s.Tables = 1 + } + if s.Columns == 0 { + s.Columns = 10 + } + if s.ResourcesPerTable == 0 { + s.ResourcesPerTable = 100 + } + if s.ResourcesPerPage == 0 { + s.ResourcesPerPage = 10 + } +} + +type ClientTest interface { + Call(clientID, tableName string) error +} + +type Benchmark struct { + *BenchmarkScenario + + b *testing.B + tables []*schema.Table + plugin *Plugin + + apiCalls atomic.Int64 +} + +func NewBenchmark(b *testing.B, scenario BenchmarkScenario) *Benchmark { + scenario.SetDefaults() + sb := &Benchmark{ + BenchmarkScenario: &scenario, + b: b, + tables: nil, + plugin: nil, + } + sb.setup(b) + return sb +} + +func (s *Benchmark) setup(b *testing.B) { + createResolvers := func(tableName string) (schema.TableResolver, schema.RowResolver, schema.ColumnResolver) { + tableResolver := func(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { + total := 0 + for total < s.ResourcesPerTable { + s.simulateAPICall(meta.ID(), tableName) + num := min(s.ResourcesPerPage, s.ResourcesPerTable-total) + resources := make([]struct { + Column1 string + }, num) + for i := 0; i < num; i++ { + resources[i] = struct { + Column1 string + }{ + Column1: "test-column", + } + } + res <- resources + total += num + } + return nil + } + preResourceResolver := func(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource) error { + s.simulateAPICall(meta.ID(), tableName) + resource.Item = struct { + Column1 string + }{ + Column1: "test-pre", + } + return nil + } + columnResolver := func(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { + s.simulateAPICall(meta.ID(), tableName) + return resource.Set(c.Name, "test") + } + return tableResolver, preResourceResolver, columnResolver + } + + s.tables = make([]*schema.Table, s.Tables) + for i := 0; i < s.Tables; i++ { + tableResolver, preResourceResolver, columnResolver := createResolvers(fmt.Sprintf("table%d", i)) + columns := make([]schema.Column, s.Columns) + for u := 0; u < s.Columns; u++ { + columns[u] = schema.Column{ + Name: fmt.Sprintf("column%d", u), + Type: arrow.BinaryTypes.String, + } + if u < s.ColumnResolvers { + columns[u].Resolver = columnResolver + } + } + relations := make([]*schema.Table, s.ChildrenPerTable) + for u := 0; u < s.ChildrenPerTable; u++ { + relations[u] = &schema.Table{ + Name: fmt.Sprintf("table%d_child%d", i, u), + Columns: columns, + Resolver: tableResolver, + } + if !s.NoPreResourceResolver { + relations[u].PreResourceResolver = preResourceResolver + } + } + s.tables[i] = &schema.Table{ + Name: fmt.Sprintf("table%d", i), + Columns: columns, + Relations: relations, + Resolver: tableResolver, + Multiplex: nMultiplexer(s.Clients), + } + if !s.NoPreResourceResolver { + s.tables[i].PreResourceResolver = preResourceResolver + } + for u := range relations { + relations[u].Parent = s.tables[i] + } + } + + plugin := NewPlugin( + "testPlugin", + "1.0.0", + s.tables, + newTestExecutionClient, + ) + plugin.SetLogger(zerolog.New(zerolog.NewTestWriter(b)).Level(zerolog.WarnLevel)) + s.plugin = plugin + s.b = b +} + +func (s *Benchmark) simulateAPICall(clientID, tableName string) { + for { + s.apiCalls.Add(1) + err := s.Client.Call(clientID, tableName) + if err == nil { + // if no error, we are done + break + } + // if error, we have to retry + // we simulate a random backoff + time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond) + } +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} + +func (s *Benchmark) Run() { + for n := 0; n < s.b.N; n++ { + s.b.StopTimer() + ctx := context.Background() + spec := specs.Source{ + Name: "testSource", + Path: "cloudquery/testSource", + Tables: []string{"*"}, + Version: "v1.0.0", + Destinations: []string{"test"}, + Concurrency: s.Concurrency, + Scheduler: s.Scheduler, + } + if err := s.plugin.Init(ctx, spec); err != nil { + s.b.Fatal(err) + } + resources := make(chan *schema.Resource) + g, ctx := errgroup.WithContext(ctx) + g.Go(func() error { + defer close(resources) + return s.plugin.Sync(ctx, + time.Now(), + resources) + }) + s.b.StartTimer() + start := time.Now() + + totalResources := 0 + for range resources { + // read resources channel until empty + totalResources++ + } + if err := g.Wait(); err != nil { + s.b.Fatal(err) + } + + end := time.Now() + s.b.ReportMetric(0, "ns/op") // drop default ns/op output + s.b.ReportMetric(float64(totalResources)/(end.Sub(start).Seconds()), "resources/s") + + // Enable the below metrics for more verbose information about the scenario: + // s.b.ReportMetric(float64(s.apiCalls.Load())/(end.Sub(start).Seconds()), "api-calls/s") + // s.b.ReportMetric(float64(totalResources), "resources") + // s.b.ReportMetric(float64(s.apiCalls.Load()), "apiCalls") + } +} + +type benchmarkClient struct { + num int +} + +func (b benchmarkClient) ID() string { + return fmt.Sprintf("client%d", b.num) +} + +func nMultiplexer(n int) schema.Multiplexer { + return func(meta schema.ClientMeta) []schema.ClientMeta { + clients := make([]schema.ClientMeta, n) + for i := 0; i < n; i++ { + clients[i] = benchmarkClient{ + num: i, + } + } + return clients + } +} + +func BenchmarkDefaultConcurrencyDFS(b *testing.B) { + benchmarkWithScheduler(b, specs.SchedulerDFS) +} + +func BenchmarkDefaultConcurrencyRoundRobin(b *testing.B) { + benchmarkWithScheduler(b, specs.SchedulerRoundRobin) +} + +func benchmarkWithScheduler(b *testing.B, scheduler specs.Scheduler) { + b.ReportAllocs() + minTime := 1 * time.Millisecond + mean := 10 * time.Millisecond + stdDev := 100 * time.Millisecond + client := NewDefaultClient(minTime, mean, stdDev) + bs := BenchmarkScenario{ + Client: client, + Clients: 25, + Tables: 5, + Columns: 10, + ColumnResolvers: 1, + ResourcesPerTable: 100, + ResourcesPerPage: 50, + Scheduler: scheduler, + } + sb := NewBenchmark(b, bs) + sb.Run() +} + +func BenchmarkTablesWithChildrenDFS(b *testing.B) { + benchmarkTablesWithChildrenScheduler(b, specs.SchedulerDFS) +} + +func BenchmarkTablesWithChildrenRoundRobin(b *testing.B) { + benchmarkTablesWithChildrenScheduler(b, specs.SchedulerRoundRobin) +} + +func benchmarkTablesWithChildrenScheduler(b *testing.B, scheduler specs.Scheduler) { + b.ReportAllocs() + minTime := 1 * time.Millisecond + mean := 10 * time.Millisecond + stdDev := 100 * time.Millisecond + client := NewDefaultClient(minTime, mean, stdDev) + bs := BenchmarkScenario{ + Client: client, + Clients: 2, + Tables: 2, + ChildrenPerTable: 2, + Columns: 10, + ColumnResolvers: 1, + ResourcesPerTable: 100, + ResourcesPerPage: 50, + Scheduler: scheduler, + } + sb := NewBenchmark(b, bs) + sb.Run() +} + +type DefaultClient struct { + min, stdDev, mean time.Duration +} + +func NewDefaultClient(min, mean, stdDev time.Duration) *DefaultClient { + if min == 0 { + min = time.Millisecond + } + if mean == 0 { + mean = 10 * time.Millisecond + } + if stdDev == 0 { + stdDev = 100 * time.Millisecond + } + return &DefaultClient{ + min: min, + mean: mean, + stdDev: stdDev, + } +} + +func (c *DefaultClient) Call(_, _ string) error { + sample := int(rand.NormFloat64()*float64(c.stdDev) + float64(c.mean)) + duration := time.Duration(sample) + if duration < c.min { + duration = c.min + } + time.Sleep(duration) + return nil +} + +type RateLimitClient struct { + *DefaultClient + calls map[string][]time.Time + callsLock sync.Mutex + window time.Duration + maxCallsPerWindow int +} + +func NewRateLimitClient(min, mean, stdDev time.Duration, maxCallsPerWindow int, window time.Duration) *RateLimitClient { + return &RateLimitClient{ + DefaultClient: NewDefaultClient(min, mean, stdDev), + calls: map[string][]time.Time{}, + window: window, + maxCallsPerWindow: maxCallsPerWindow, + } +} + +func (r *RateLimitClient) Call(clientID, table string) error { + // this will sleep for the appropriate amount of time before responding + err := r.DefaultClient.Call(clientID, table) + if err != nil { + return err + } + + r.callsLock.Lock() + defer r.callsLock.Unlock() + + // limit the number of calls per window by table + key := table + + // remove calls from outside the call window + updated := make([]time.Time, 0, len(r.calls[key])) + for i := range r.calls[key] { + if time.Since(r.calls[key][i]) < r.window { + updated = append(updated, r.calls[key][i]) + } + } + + // return error if we've exceeded the max calls in the time window + if len(updated) >= r.maxCallsPerWindow { + return fmt.Errorf("rate limit exceeded") + } + + r.calls[key] = append(r.calls[key], time.Now()) + return nil +} + +// BenchmarkDefaultConcurrency represents a benchmark scenario where rate limiting is applied +// by the cloud provider. In this rate limiter, the limit is applied globally per table. +// This mirrors the behavior of GCP, where rate limiting is applied per project *token*, not +// per project. A good scheduler should spread the load across tables so that other tables can make +// progress while waiting for the rate limit to reset. +func BenchmarkTablesWithRateLimitingDFS(b *testing.B) { + benchmarkTablesWithRateLimitingScheduler(b, specs.SchedulerDFS) +} + +func BenchmarkTablesWithRateLimitingRoundRobin(b *testing.B) { + benchmarkTablesWithRateLimitingScheduler(b, specs.SchedulerRoundRobin) +} + +// In this benchmark, we set up a scenario where each table has a global rate limit of 1 call per 100ms. +// Every table requires 1 call to resolve, and has 10 clients. This means, at best, each table can resolve in 1 second. +// We have 100 such tables and a concurrency that allows 1000 calls at a time. A good scheduler for this scenario +// should be able to resolve all tables in a bit more than 1 second. +func benchmarkTablesWithRateLimitingScheduler(b *testing.B, scheduler specs.Scheduler) { + b.ReportAllocs() + minTime := 1 * time.Millisecond + mean := 1 * time.Millisecond + stdDev := 1 * time.Millisecond + maxCallsPerWindow := 1 + window := 100 * time.Millisecond + c := NewRateLimitClient(minTime, mean, stdDev, maxCallsPerWindow, window) + + bs := BenchmarkScenario{ + Client: c, + Scheduler: scheduler, + Clients: 10, + Tables: 100, + ChildrenPerTable: 0, + Columns: 10, + ColumnResolvers: 0, + ResourcesPerTable: 1, + ResourcesPerPage: 1, + Concurrency: 1000, + NoPreResourceResolver: true, + } + sb := NewBenchmark(b, bs) + sb.Run() +} diff --git a/plugin-sdk/scheduler/metrics.go b/plugin-sdk/scheduler/metrics.go new file mode 100644 index 00000000..82b6fff5 --- /dev/null +++ b/plugin-sdk/scheduler/metrics.go @@ -0,0 +1,247 @@ +package scheduler + +import ( + "context" + "sync/atomic" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +// Metrics is deprecated as we move toward open telemetry for tracing and metrics +type Metrics struct { + TableClient map[string]map[string]*TableClientMetrics +} + +type OtelMeters struct { + resources metric.Int64Counter + errors metric.Int64Counter + panics metric.Int64Counter + startTime metric.Int64Counter + endTime metric.Int64Counter + attributes []attribute.KeyValue +} + +type TableClientMetrics struct { + Resources uint64 + Errors uint64 + Panics uint64 + Duration atomic.Pointer[time.Duration] + + otelMeters *OtelMeters +} + +func durationPointerEqual(a, b *time.Duration) bool { + if a == nil { + return b == nil + } + return b != nil && *a == *b +} + +func (m *TableClientMetrics) Equal(other *TableClientMetrics) bool { + return m.Resources == other.Resources && m.Errors == other.Errors && m.Panics == other.Panics && durationPointerEqual(m.Duration.Load(), other.Duration.Load()) +} + +// Equal compares to stats. Mostly useful in testing +func (s *Metrics) Equal(other *Metrics) bool { + for table, clientStats := range s.TableClient { + for client, stats := range clientStats { + if _, ok := other.TableClient[table]; !ok { + return false + } + if _, ok := other.TableClient[table][client]; !ok { + return false + } + if !stats.Equal(other.TableClient[table][client]) { + return false + } + } + } + for table, clientStats := range other.TableClient { + for client, stats := range clientStats { + if _, ok := s.TableClient[table]; !ok { + return false + } + if _, ok := s.TableClient[table][client]; !ok { + return false + } + if !stats.Equal(s.TableClient[table][client]) { + return false + } + } + } + return true +} + +func getOtelMeters(tableName string, clientID string) *OtelMeters { + resources, err := otel.Meter(otelName).Int64Counter("sync.table.resources", + metric.WithDescription("Number of resources synced for a table"), + metric.WithUnit("/{tot}"), + ) + if err != nil { + return nil + } + + errors, err := otel.Meter(otelName).Int64Counter("sync.table.errors", + metric.WithDescription("Number of errors encountered while syncing a table"), + metric.WithUnit("/{tot}"), + ) + if err != nil { + return nil + } + + panics, err := otel.Meter(otelName).Int64Counter("sync.table.panics", + metric.WithDescription("Number of panics encountered while syncing a table"), + metric.WithUnit("/{tot}"), + ) + if err != nil { + return nil + } + + startTime, err := otel.Meter(otelName).Int64Counter("sync.table.start_time", + metric.WithDescription("Start time of syncing a table"), + metric.WithUnit("ns"), + ) + if err != nil { + return nil + } + + endTime, err := otel.Meter(otelName).Int64Counter("sync.table.end_time", + metric.WithDescription("End time of syncing a table"), + metric.WithUnit("ns"), + ) + + if err != nil { + return nil + } + + return &OtelMeters{ + resources: resources, + errors: errors, + panics: panics, + startTime: startTime, + endTime: endTime, + attributes: []attribute.KeyValue{ + attribute.Key("sync.client.id").String(clientID), + attribute.Key("sync.table.name").String(tableName), + }, + } +} + +func (s *Metrics) initWithClients(table *schema.Table, clients []schema.ClientMeta) { + s.TableClient[table.Name] = make(map[string]*TableClientMetrics, len(clients)) + for _, client := range clients { + tableName := table.Name + clientID := client.ID() + s.TableClient[tableName][clientID] = &TableClientMetrics{ + otelMeters: getOtelMeters(tableName, clientID), + } + } + for _, relation := range table.Relations { + s.initWithClients(relation, clients) + } +} + +func (s *Metrics) TotalErrors() uint64 { + var total uint64 + for _, clientMetrics := range s.TableClient { + for _, metrics := range clientMetrics { + total += metrics.Errors + } + } + return total +} + +func (s *Metrics) TotalErrorsAtomic() uint64 { + var total uint64 + for _, clientMetrics := range s.TableClient { + for _, metrics := range clientMetrics { + total += atomic.LoadUint64(&metrics.Errors) + } + } + return total +} + +func (s *Metrics) TotalPanics() uint64 { + var total uint64 + for _, clientMetrics := range s.TableClient { + for _, metrics := range clientMetrics { + total += metrics.Panics + } + } + return total +} + +func (s *Metrics) TotalPanicsAtomic() uint64 { + var total uint64 + for _, clientMetrics := range s.TableClient { + for _, metrics := range clientMetrics { + total += atomic.LoadUint64(&metrics.Panics) + } + } + return total +} + +func (s *Metrics) TotalResources() uint64 { + var total uint64 + for _, clientMetrics := range s.TableClient { + for _, metrics := range clientMetrics { + total += metrics.Resources + } + } + return total +} + +func (s *Metrics) TotalResourcesAtomic() uint64 { + var total uint64 + for _, clientMetrics := range s.TableClient { + for _, metrics := range clientMetrics { + total += atomic.LoadUint64(&metrics.Resources) + } + } + return total +} + +func (m *TableClientMetrics) OtelResourcesAdd(ctx context.Context, count int64) { + if m.otelMeters == nil { + return + } + + m.otelMeters.resources.Add(ctx, count, metric.WithAttributes(m.otelMeters.attributes...)) +} + +func (m *TableClientMetrics) OtelErrorsAdd(ctx context.Context, count int64) { + if m.otelMeters == nil { + return + } + + m.otelMeters.errors.Add(ctx, count, metric.WithAttributes(m.otelMeters.attributes...)) +} + +func (m *TableClientMetrics) OtelPanicsAdd(ctx context.Context, count int64) { + if m.otelMeters == nil { + return + } + + m.otelMeters.panics.Add(ctx, count, metric.WithAttributes(m.otelMeters.attributes...)) +} + +func (m *TableClientMetrics) OtelStartTime(ctx context.Context, start time.Time) { + if m.otelMeters == nil { + return + } + + m.otelMeters.startTime.Add(ctx, start.UnixNano(), metric.WithAttributes(m.otelMeters.attributes...)) +} + +func (m *TableClientMetrics) OtelEndTime(ctx context.Context, end time.Time) { + if m.otelMeters == nil { + return + } + + m.otelMeters.endTime.Add(ctx, end.UnixNano(), metric.WithAttributes(m.otelMeters.attributes...)) +} diff --git a/plugin-sdk/scheduler/metrics_test.go b/plugin-sdk/scheduler/metrics_test.go new file mode 100644 index 00000000..1bc11daa --- /dev/null +++ b/plugin-sdk/scheduler/metrics_test.go @@ -0,0 +1,37 @@ +package scheduler + +import "testing" + +func TestMetrics(t *testing.T) { + s := &Metrics{ + TableClient: make(map[string]map[string]*TableClientMetrics), + } + s.TableClient["test_table"] = make(map[string]*TableClientMetrics) + s.TableClient["test_table"]["testExecutionClient"] = &TableClientMetrics{ + Resources: 1, + Errors: 2, + Panics: 3, + } + if s.TotalResources() != 1 { + t.Fatal("expected 1 resource") + } + if s.TotalErrors() != 2 { + t.Fatal("expected 2 error") + } + if s.TotalPanics() != 3 { + t.Fatal("expected 3 panics") + } + + other := &Metrics{ + TableClient: make(map[string]map[string]*TableClientMetrics), + } + other.TableClient["test_table"] = make(map[string]*TableClientMetrics) + other.TableClient["test_table"]["testExecutionClient"] = &TableClientMetrics{ + Resources: 1, + Errors: 2, + Panics: 3, + } + if !s.Equal(other) { + t.Fatal("expected metrics to be equal") + } +} diff --git a/plugin-sdk/scheduler/scheduler.go b/plugin-sdk/scheduler/scheduler.go new file mode 100644 index 00000000..e8e90232 --- /dev/null +++ b/plugin-sdk/scheduler/scheduler.go @@ -0,0 +1,350 @@ +package scheduler + +import ( + "context" + "errors" + "fmt" + "runtime/debug" + "sync" + "sync/atomic" + "time" + + "github.com/rs/zerolog" + "github.com/thoas/go-funk" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "golang.org/x/sync/semaphore" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/caser" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +const ( + DefaultSingleResourceMaxConcurrency = 5 + DefaultSingleNestedTableMaxConcurrency = 5 + DefaultConcurrency = 50000 + DefaultMaxDepth = 4 + minTableConcurrency = 1 + minResourceConcurrency = 100 + otelName = "io.cloudquery" +) + +var ErrNoTables = errors.New("no tables specified for syncing, review `tables` and `skip_tables` in your config and specify at least one table to sync") + +const ( + StrategyDFS Strategy = iota + StrategyRoundRobin + StrategyShuffle +) + +type Option func(*Scheduler) + +func WithLogger(logger zerolog.Logger) Option { + return func(s *Scheduler) { + s.logger = logger + } +} + +func WithConcurrency(concurrency int) Option { + return func(s *Scheduler) { + s.concurrency = concurrency + } +} + +func WithMaxDepth(maxDepth uint64) Option { + return func(s *Scheduler) { + s.maxDepth = maxDepth + } +} + +func WithStrategy(strategy Strategy) Option { + return func(s *Scheduler) { + s.strategy = strategy + } +} + +func WithSingleNestedTableMaxConcurrency(concurrency int64) Option { + return func(s *Scheduler) { + s.singleNestedTableMaxConcurrency = concurrency + } +} + +func WithSingleResourceMaxConcurrency(concurrency int64) Option { + return func(s *Scheduler) { + s.singleResourceMaxConcurrency = concurrency + } +} + +type SyncOption func(*syncClient) + +func WithSyncDeterministicCQID(deterministicCQID bool) SyncOption { + return func(s *syncClient) { + s.deterministicCQID = deterministicCQID + } +} + +func WithInvocationID(invocationID string) Option { + return func(s *Scheduler) { + s.invocationID = invocationID + } +} + +type Client interface { + ID() string +} + +type Scheduler struct { + caser *caser.Caser + strategy Strategy + maxDepth uint64 + // resourceSem is a semaphore that limits the number of concurrent resources being fetched + resourceSem *semaphore.Weighted + // tableSem is a semaphore that limits the number of concurrent tables being fetched + tableSems []*semaphore.Weighted + // Logger to call, this logger is passed to the serve.Serve Client, if not defined Serve will create one instead. + logger zerolog.Logger + concurrency int + // This Map holds all of the concurrency semaphores for each table+client pair. + singleTableConcurrency sync.Map + // The maximum number of go routines that can be spawned for a single table+client pair + singleNestedTableMaxConcurrency int64 + + // The maximum number of go routines that can be spawned for a specific resource + singleResourceMaxConcurrency int64 + + // Controls how records are constructed on the source side. + batchSettings *BatchSettings + + invocationID string +} + +type syncClient struct { + tables schema.Tables + client schema.ClientMeta + scheduler *Scheduler + deterministicCQID bool + // status sync metrics + metrics *Metrics + logger zerolog.Logger + invocationID string +} + +func NewScheduler(opts ...Option) *Scheduler { + s := Scheduler{ + caser: caser.New(), + concurrency: DefaultConcurrency, + maxDepth: DefaultMaxDepth, + singleResourceMaxConcurrency: DefaultSingleResourceMaxConcurrency, + singleNestedTableMaxConcurrency: DefaultSingleNestedTableMaxConcurrency, + batchSettings: &BatchSettings{ + MaxRows: DefaultBatchMaxRows, + Timeout: DefaultBatchTimeout, + }, + } + for _, opt := range opts { + opt(&s) + } + // This is very similar to the concurrent web crawler problem with some minor changes. + // We are using DFS/Round-Robin to make sure memory usage is capped at O(h) where h is the height of the tree. + tableConcurrency := max(s.concurrency/minResourceConcurrency, minTableConcurrency) + resourceConcurrency := tableConcurrency * minResourceConcurrency + s.tableSems = make([]*semaphore.Weighted, s.maxDepth) + for i := uint64(0); i < s.maxDepth; i++ { + s.tableSems[i] = semaphore.NewWeighted(int64(tableConcurrency)) + // reduce table concurrency logarithmically for every depth level + tableConcurrency = max(tableConcurrency/2, minTableConcurrency) + } + s.resourceSem = semaphore.NewWeighted(int64(resourceConcurrency)) + + return &s +} + +// SyncAll is mostly used for testing as it will sync all tables and can run out of memory +// in the real world. Should use Sync for production. +func (s *Scheduler) SyncAll(ctx context.Context, client schema.ClientMeta, tables schema.Tables) (message.SyncMessages, error) { + res := make(chan message.SyncMessage) + resourceCh := make(chan *schema.Resource) + var err error + go func() { + defer close(res) + defer close(resourceCh) + err = s.Sync(ctx, client, tables, res, resourceCh) + }() + // nolint:prealloc + var messages message.SyncMessages + for msg := range res { + messages = append(messages, msg) + } + return messages, err +} + +func (s *Scheduler) Sync(ctx context.Context, client schema.ClientMeta, tables schema.Tables, res chan<- message.SyncMessage, resourceCh chan<- *schema.Resource, opts ...SyncOption) error { + ctx, span := otel.Tracer(otelName).Start(ctx, + "sync", + trace.WithAttributes(attribute.Key("sync.invocation.id").String(s.invocationID)), + ) + defer span.End() + if len(tables) == 0 { + return ErrNoTables + } + + syncClient := &syncClient{ + metrics: &Metrics{TableClient: make(map[string]map[string]*TableClientMetrics)}, + tables: tables, + client: client, + scheduler: s, + logger: s.logger, + invocationID: s.invocationID, + } + for _, opt := range opts { + opt(syncClient) + } + + if maxDepth(tables) > s.maxDepth { + return fmt.Errorf("max depth exceeded, max depth is %d", s.maxDepth) + } + + // send migrate messages first + for _, tableOriginal := range tables.FlattenTables() { + migrateMessage := &message.SyncMigrateTable{ + Table: tableOriginal.Copy(tableOriginal.Parent), + } + if syncClient.deterministicCQID { + schema.CqIDAsPK(migrateMessage.Table) + } + res <- migrateMessage + } + + resources := make(chan *schema.Resource) + go func() { + defer close(resources) + switch s.strategy { + case StrategyDFS: + syncClient.syncDfs(ctx, resources) + case StrategyRoundRobin: + syncClient.syncRoundRobin(ctx, resources) + case StrategyShuffle: + syncClient.syncShuffle(ctx, resources) + default: + panic(fmt.Errorf("unknown scheduler %s", s.strategy.String())) + } + }() + + b := s.batchSettings.getBatcher(ctx, res, s.logger) + defer b.close() // wait for all resources to be processed + done := ctx.Done() // no need to do the lookups in loop + for resource := range resources { + select { + case <-done: + s.logger.Debug().Msg("sync context cancelled") + return context.Cause(ctx) + default: + b.process(resource) + } + + select { + case resourceCh <- resource: + default: + s.logger.Error().Msg("failed to publish resource to channel") + } + } + + return context.Cause(ctx) +} + +func (s *syncClient) logTablesMetrics(tables schema.Tables, client Client) { + clientName := client.ID() + for _, table := range tables { + metrics := s.metrics.TableClient[table.Name][clientName] + duration := metrics.Duration.Load() + if duration == nil { + // This can happen for a relation when there are no resources to resolve from the parent + duration = new(time.Duration) + } + s.logger.Debug().Str("table", table.Name).Str("client", clientName).Uint64("resources", metrics.Resources).Dur("duration_ms", *duration).Uint64("errors", metrics.Errors).Msg("table sync finished") + s.logTablesMetrics(table.Relations, client) + } +} + +func (s *syncClient) resolveResource(ctx context.Context, table *schema.Table, client schema.ClientMeta, parent *schema.Resource, item any) *schema.Resource { + ctx, cancel := context.WithTimeout(ctx, 10*time.Minute) + defer cancel() + resource := schema.NewResourceData(table, parent, item) + objectStartTime := time.Now() + clientID := client.ID() + tableMetrics := s.metrics.TableClient[table.Name][clientID] + logger := s.logger.With().Str("table", table.Name).Str("client", clientID).Logger() + defer func() { + if err := recover(); err != nil { + stack := fmt.Sprintf("%s\n%s", err, string(debug.Stack())) + logger.Error().Interface("error", err).TimeDiff("duration", time.Now(), objectStartTime).Str("stack", stack).Msg("resource resolver finished with panic") + atomic.AddUint64(&tableMetrics.Panics, 1) + } + }() + if table.PreResourceResolver != nil { + if err := table.PreResourceResolver(ctx, client, resource); err != nil { + logger.Error().Err(err).Msg("pre resource resolver failed") + atomic.AddUint64(&tableMetrics.Errors, 1) + return nil + } + } + + for _, c := range table.Columns { + s.resolveColumn(ctx, logger, tableMetrics, client, resource, c) + } + + if table.PostResourceResolver != nil { + if err := table.PostResourceResolver(ctx, client, resource); err != nil { + logger.Error().Stack().Err(err).Msg("post resource resolver finished with error") + atomic.AddUint64(&tableMetrics.Errors, 1) + } + } + + tableMetrics.OtelResourcesAdd(ctx, 1) + atomic.AddUint64(&tableMetrics.Resources, 1) + return resource +} + +func (s *syncClient) resolveColumn(ctx context.Context, logger zerolog.Logger, tableMetrics *TableClientMetrics, client schema.ClientMeta, resource *schema.Resource, c schema.Column) { + columnStartTime := time.Now() + defer func() { + if err := recover(); err != nil { + stack := fmt.Sprintf("%s\n%s", err, string(debug.Stack())) + logger.Error().Str("column", c.Name).Interface("error", err).TimeDiff("duration", time.Now(), columnStartTime).Str("stack", stack).Msg("column resolver finished with panic") + atomic.AddUint64(&tableMetrics.Panics, 1) + } + }() + + if c.Resolver != nil { + if err := c.Resolver(ctx, client, resource, c); err != nil { + logger.Error().Err(err).Msg("column resolver finished with error") + atomic.AddUint64(&tableMetrics.Errors, 1) + } + } else { + // base use case: try to get column with CamelCase name + v := funk.Get(resource.GetItem(), s.scheduler.caser.ToPascal(c.Name), funk.WithAllowZero()) + if v != nil { + err := resource.Set(c.Name, v) + if err != nil { + logger.Error().Err(err).Msg("column resolver finished with error") + atomic.AddUint64(&tableMetrics.Errors, 1) + } + } + } +} + +func maxDepth(tables schema.Tables) uint64 { + var depth uint64 + if len(tables) == 0 { + return 0 + } + for _, table := range tables { + newDepth := 1 + maxDepth(table.Relations) + if newDepth > depth { + depth = newDepth + } + } + return depth +} diff --git a/plugin-sdk/scheduler/scheduler_dfs.go b/plugin-sdk/scheduler/scheduler_dfs.go new file mode 100644 index 00000000..e29fa81b --- /dev/null +++ b/plugin-sdk/scheduler/scheduler_dfs.go @@ -0,0 +1,217 @@ +package scheduler + +import ( + "context" + "fmt" + "runtime/debug" + "sync" + "sync/atomic" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "golang.org/x/sync/semaphore" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/helpers" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +func (s *syncClient) syncDfs(ctx context.Context, resolvedResources chan<- *schema.Resource) { + // we have this because plugins can return sometimes clients in a random way which will cause + // differences between this run and the next one. + preInitialisedClients := make([][]schema.ClientMeta, len(s.tables)) + for i, table := range s.tables { + clients := []schema.ClientMeta{s.client} + if table.Multiplex != nil { + clients = table.Multiplex(s.client) + } + // Detect duplicate clients while multiplexing + seenClients := make(map[string]bool) + for _, c := range clients { + if _, ok := seenClients[c.ID()]; !ok { + seenClients[c.ID()] = true + } else { + s.logger.Warn().Str("client", c.ID()).Str("table", table.Name).Msg("multiplex returned duplicate client") + } + } + preInitialisedClients[i] = clients + // we do this here to avoid locks so we initial the metrics structure once in the main goroutines + // and then we can just read from it in the other goroutines concurrently given we are not writing to it. + s.metrics.initWithClients(table, clients) + } + + var wg sync.WaitGroup + for i, table := range s.tables { + table := table + clients := preInitialisedClients[i] + for _, client := range clients { + client := client + if err := s.scheduler.tableSems[0].Acquire(ctx, 1); err != nil { + // This means context was cancelled + wg.Wait() + return + } + wg.Add(1) + go func() { + defer wg.Done() + defer s.scheduler.tableSems[0].Release(1) + // not checking for error here as nothing much todo. + // the error is logged and this happens when context is cancelled + s.resolveTableDfs(ctx, table, client, nil, resolvedResources, 1) + }() + } + } + + // Wait for all the worker goroutines to finish + wg.Wait() +} + +func (s *syncClient) resolveTableDfs(ctx context.Context, table *schema.Table, client schema.ClientMeta, parent *schema.Resource, resolvedResources chan<- *schema.Resource, depth int) { + clientName := client.ID() + ctx, span := otel.Tracer(otelName).Start(ctx, + "sync.table."+table.Name, + trace.WithAttributes( + attribute.Key("sync.client.id").String(clientName), + attribute.Key("sync.invocation.id").String(s.invocationID), + ), + ) + defer span.End() + logger := s.logger.With().Str("table", table.Name).Str("client", clientName).Logger() + + startTime := time.Now() + if parent == nil { // Log only for root tables, otherwise we spam too much. + logger.Debug().Msg("top level table resolver started") + } + tableMetrics := s.metrics.TableClient[table.Name][clientName] + tableMetrics.OtelStartTime(ctx, startTime) + + res := make(chan any) + go func() { + defer func() { + if err := recover(); err != nil { + stack := fmt.Sprintf("%s\n%s", err, string(debug.Stack())) + logger.Error().Interface("error", err).Str("stack", stack).Msg("table resolver finished with panic") + tableMetrics.OtelPanicsAdd(ctx, 1) + atomic.AddUint64(&tableMetrics.Panics, 1) + } + close(res) + }() + if err := table.Resolver(ctx, client, parent, res); err != nil { + logger.Error().Err(err).Msg("table resolver finished with error") + tableMetrics.OtelErrorsAdd(ctx, 1) + atomic.AddUint64(&tableMetrics.Errors, 1) + return + } + }() + + for r := range res { + s.resolveResourcesDfs(ctx, table, client, parent, r, resolvedResources, depth) + } + + // we don't need any waitgroups here because we are waiting for the channel to close + endTime := time.Now() + duration := endTime.Sub(startTime) + tableMetrics.Duration.Store(&duration) + tableMetrics.OtelEndTime(ctx, endTime) + if parent == nil { // Log only for root tables and relations only after resolving is done, otherwise we spam per object instead of per table. + logger.Debug().Uint64("resources", tableMetrics.Resources).Uint64("errors", tableMetrics.Errors).Dur("duration_ms", duration).Msg("table sync finished") + s.logTablesMetrics(table.Relations, client) + } +} + +func (s *syncClient) resolveResourcesDfs(ctx context.Context, table *schema.Table, client schema.ClientMeta, parent *schema.Resource, resources any, resolvedResources chan<- *schema.Resource, depth int) { + resourcesSlice := helpers.InterfaceSlice(resources) + if len(resourcesSlice) == 0 { + return + } + resourcesChan := make(chan *schema.Resource, len(resourcesSlice)) + go func() { + defer close(resourcesChan) + var wg sync.WaitGroup + for i := range resourcesSlice { + i := i + resourceConcurrencyKey := table.Name + "-" + client.ID() + "-" + "resource" + resourceSemVal, _ := s.scheduler.singleTableConcurrency.LoadOrStore(resourceConcurrencyKey, semaphore.NewWeighted(s.scheduler.singleResourceMaxConcurrency)) + resourceSem := resourceSemVal.(*semaphore.Weighted) + if err := resourceSem.Acquire(ctx, 1); err != nil { + s.logger.Warn().Err(err).Msg("failed to acquire semaphore. context cancelled") + // This means context was cancelled + wg.Wait() + // we have to continue emptying the channel to exit gracefully + return + } + + // Once Resource semaphore is acquired we can acquire the global semaphore + if err := s.scheduler.resourceSem.Acquire(ctx, 1); err != nil { + // This means context was cancelled + resourceSem.Release(1) + wg.Wait() + // we have to continue emptying the channel to exit gracefully + return + } + wg.Add(1) + go func() { + defer resourceSem.Release(1) + defer s.scheduler.resourceSem.Release(1) + defer wg.Done() + //nolint:all + resolvedResource := s.resolveResource(ctx, table, client, parent, resourcesSlice[i]) + if resolvedResource == nil { + return + } + + if err := resolvedResource.CalculateCQID(s.deterministicCQID); err != nil { + tableMetrics := s.metrics.TableClient[table.Name][client.ID()] + s.logger.Error().Err(err).Str("table", table.Name).Str("client", client.ID()).Msg("resource resolver finished with primary key calculation error") + atomic.AddUint64(&tableMetrics.Errors, 1) + return + } + if err := resolvedResource.Validate(); err != nil { + tableMetrics := s.metrics.TableClient[table.Name][client.ID()] + s.logger.Error().Err(err).Str("table", table.Name).Str("client", client.ID()).Msg("resource resolver finished with validation error") + atomic.AddUint64(&tableMetrics.Errors, 1) + return + } + select { + case resourcesChan <- resolvedResource: + case <-ctx.Done(): + } + }() + } + wg.Wait() + }() + + var wg sync.WaitGroup + for resource := range resourcesChan { + resource := resource + resolvedResources <- resource + for _, relation := range resource.Table.Relations { + relation := relation + tableConcurrencyKey := table.Name + "-" + client.ID() + // Acquire the semaphore for the table + tableSemVal, _ := s.scheduler.singleTableConcurrency.LoadOrStore(tableConcurrencyKey, semaphore.NewWeighted(s.scheduler.singleNestedTableMaxConcurrency)) + tableSem := tableSemVal.(*semaphore.Weighted) + if err := tableSem.Acquire(ctx, 1); err != nil { + // This means context was cancelled + wg.Wait() + return + } + // Once table semaphore is acquired we can acquire the global semaphore + if err := s.scheduler.tableSems[depth].Acquire(ctx, 1); err != nil { + // This means context was cancelled + tableSem.Release(1) + wg.Wait() + return + } + wg.Add(1) + go func() { + defer wg.Done() + defer tableSem.Release(1) + defer s.scheduler.tableSems[depth].Release(1) + s.resolveTableDfs(ctx, relation, client, resource, resolvedResources, depth+1) + }() + } + } + wg.Wait() +} diff --git a/plugin-sdk/scheduler/scheduler_round_robin.go b/plugin-sdk/scheduler/scheduler_round_robin.go new file mode 100644 index 00000000..a10553f3 --- /dev/null +++ b/plugin-sdk/scheduler/scheduler_round_robin.go @@ -0,0 +1,76 @@ +package scheduler + +import ( + "context" + "sync" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +type tableClient struct { + table *schema.Table + client schema.ClientMeta +} + +func (s *syncClient) syncRoundRobin(ctx context.Context, resolvedResources chan<- *schema.Resource) { + // we have this because plugins can return sometimes clients in a random way which will cause + // differences between this run and the next one. + preInitialisedClients := make([][]schema.ClientMeta, len(s.tables)) + for i, table := range s.tables { + clients := []schema.ClientMeta{s.client} + if table.Multiplex != nil { + clients = table.Multiplex(s.client) + } + preInitialisedClients[i] = clients + // we do this here to avoid locks so we initial the metrics structure once in the main goroutines + // and then we can just read from it in the other goroutines concurrently given we are not writing to it. + s.metrics.initWithClients(table, clients) + } + + tableClients := roundRobinInterleave(s.tables, preInitialisedClients) + + var wg sync.WaitGroup + for _, tc := range tableClients { + table := tc.table + cl := tc.client + if err := s.scheduler.tableSems[0].Acquire(ctx, 1); err != nil { + // This means context was cancelled + wg.Wait() + return + } + wg.Add(1) + go func() { + defer wg.Done() + defer s.scheduler.tableSems[0].Release(1) + // not checking for error here as nothing much to do. + // the error is logged and this happens when context is cancelled + // Round Robin currently uses the DFS algorithm to resolve the tables, but this + // may change in the future. + s.resolveTableDfs(ctx, table, cl, nil, resolvedResources, 1) + }() + } + + // Wait for all the worker goroutines to finish + wg.Wait() +} + +// interleave table-clients so that we get: +// table1-client1, table2-client1, table3-client1, table1-client2, table2-client2, table3-client2, ... +func roundRobinInterleave(tables schema.Tables, preInitialisedClients [][]schema.ClientMeta) []tableClient { + tableClients := make([]tableClient, 0) + c := 0 + for { + addedNew := false + for i, table := range tables { + if c < len(preInitialisedClients[i]) { + tableClients = append(tableClients, tableClient{table: table, client: preInitialisedClients[i][c]}) + addedNew = true + } + } + c++ + if !addedNew { + break + } + } + return tableClients +} diff --git a/plugin-sdk/scheduler/scheduler_round_robin_test.go b/plugin-sdk/scheduler/scheduler_round_robin_test.go new file mode 100644 index 00000000..c5bb8408 --- /dev/null +++ b/plugin-sdk/scheduler/scheduler_round_robin_test.go @@ -0,0 +1,65 @@ +package scheduler + +import ( + "testing" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +func TestRoundRobinInterleave(t *testing.T) { + table1 := &schema.Table{Name: "test_table"} + table2 := &schema.Table{Name: "test_table2"} + client1 := &testExecutionClient{} + client2 := &testExecutionClient{} + client3 := &testExecutionClient{} + cases := []struct { + name string + tables schema.Tables + preInitialisedClients [][]schema.ClientMeta + want []tableClient + }{ + { + name: "single table", + tables: schema.Tables{table1}, + preInitialisedClients: [][]schema.ClientMeta{{client1}}, + want: []tableClient{{table: table1, client: client1}}, + }, + { + name: "two tables with different clients", + tables: schema.Tables{table1, table2}, + preInitialisedClients: [][]schema.ClientMeta{{client1}, {client1, client2}}, + want: []tableClient{ + {table: table1, client: client1}, + {table: table2, client: client1}, + {table: table2, client: client2}, + }, + }, + { + name: "two tables with different clients", + tables: schema.Tables{table1, table2}, + preInitialisedClients: [][]schema.ClientMeta{{client1, client3}, {client1, client2}}, + want: []tableClient{ + {table: table1, client: client1}, + {table: table2, client: client1}, + {table: table1, client: client3}, + {table: table2, client: client2}, + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := roundRobinInterleave(tc.tables, tc.preInitialisedClients) + if len(got) != len(tc.want) { + t.Fatalf("got %d tableClients, want %d", len(got), len(tc.want)) + } + for i := range got { + if got[i].table != tc.want[i].table { + t.Errorf("got table %v, want %v", got[i].table, tc.want[i].table) + } + if got[i].client != tc.want[i].client { + t.Errorf("got client %v, want %v", got[i].client, tc.want[i].client) + } + } + }) + } +} diff --git a/plugin-sdk/scheduler/scheduler_shuffle.go b/plugin-sdk/scheduler/scheduler_shuffle.go new file mode 100644 index 00000000..5422c806 --- /dev/null +++ b/plugin-sdk/scheduler/scheduler_shuffle.go @@ -0,0 +1,76 @@ +package scheduler + +import ( + "context" + "hash/fnv" + "math/rand" + "strings" + "sync" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +func (s *syncClient) syncShuffle(ctx context.Context, resolvedResources chan<- *schema.Resource) { + // We have this because plugins can return sometimes clients in a random way which will cause + // differences between this run and the next one. + preInitialisedClients := make([][]schema.ClientMeta, len(s.tables)) + tableNames := make([]string, len(s.tables)) + for i, table := range s.tables { + tableNames[i] = table.Name + clients := []schema.ClientMeta{s.client} + if table.Multiplex != nil { + clients = table.Multiplex(s.client) + } + preInitialisedClients[i] = clients + // we do this here to avoid locks so we initial the metrics structure once in the main goroutines + // and then we can just read from it in the other goroutines concurrently given we are not writing to it. + s.metrics.initWithClients(table, clients) + } + + // First interleave the tables like in round-robin + tableClients := roundRobinInterleave(s.tables, preInitialisedClients) + + // Then shuffle the tableClients to randomize the order in which they are retrieved. + // We use a fixed seed so that runs with the same tables and clients perform similarly across syncs + // however, if the table order changes, the seed will change and the shuffle order will be different, + // so users have a little bit of control over the randomization. + seed := hashTableNames(tableNames) + shuffle(tableClients, seed) + + var wg sync.WaitGroup + for _, tc := range tableClients { + table := tc.table + cl := tc.client + if err := s.scheduler.tableSems[0].Acquire(ctx, 1); err != nil { + // This means context was cancelled + wg.Wait() + return + } + wg.Add(1) + go func() { + defer wg.Done() + defer s.scheduler.tableSems[0].Release(1) + // Not checking for error here as nothing much to do. + // the error is logged and this happens when context is cancelled. + // This currently uses the DFS algorithm to resolve the tables, but this + // may change in the future. + s.resolveTableDfs(ctx, table, cl, nil, resolvedResources, 1) + }() + } + + // Wait for all the worker goroutines to finish + wg.Wait() +} + +func hashTableNames(tableNames []string) int64 { + h := fnv.New32a() + h.Write([]byte(strings.Join(tableNames, ","))) + return int64(h.Sum32()) +} + +func shuffle(tableClients []tableClient, seed int64) { + r := rand.New(rand.NewSource(seed)) + r.Shuffle(len(tableClients), func(i, j int) { + tableClients[i], tableClients[j] = tableClients[j], tableClients[i] + }) +} diff --git a/plugin-sdk/scheduler/scheduler_test.go b/plugin-sdk/scheduler/scheduler_test.go new file mode 100644 index 00000000..c197c3d5 --- /dev/null +++ b/plugin-sdk/scheduler/scheduler_test.go @@ -0,0 +1,489 @@ +package scheduler + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/scalar" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +type testExecutionClient struct { +} + +func (*testExecutionClient) ID() string { + return "test" +} + +var _ schema.ClientMeta = &testExecutionClient{} + +func testResolverSuccess(_ context.Context, _ schema.ClientMeta, _ *schema.Resource, res chan<- any) error { + res <- map[string]any{ + "TestColumn": 3, + } + return nil +} + +func testResolverPanic(context.Context, schema.ClientMeta, *schema.Resource, chan<- any) error { + panic("Resolver") +} + +func testPreResourceResolverPanic(context.Context, schema.ClientMeta, *schema.Resource) error { + panic("PreResourceResolver") +} + +func testColumnResolverPanic(context.Context, schema.ClientMeta, *schema.Resource, schema.Column) error { + panic("ColumnResolver") +} + +func testTableSuccessWithData(data []any) *schema.Table { + return &schema.Table{ + Name: "test_table_success_with_data", + Resolver: func(_ context.Context, _ schema.ClientMeta, _ *schema.Resource, res chan<- any) error { + res <- data + return nil + }, + Columns: []schema.Column{ + { + Name: "test_column", + Type: arrow.PrimitiveTypes.Int64, + }, + }, + } +} + +func testTableSuccess() *schema.Table { + return &schema.Table{ + Name: "test_table_success", + Resolver: testResolverSuccess, + Columns: []schema.Column{ + { + Name: "test_column", + Type: arrow.PrimitiveTypes.Int64, + }, + }, + } +} + +func testTableSuccessWithPK() *schema.Table { + return &schema.Table{ + Name: "test_table_success_pk", + Resolver: testResolverSuccess, + Columns: []schema.Column{ + { + Name: "test_column", + Type: arrow.PrimitiveTypes.Int64, + PrimaryKey: true, + }, + }, + } +} + +func testTableSuccessWithCQIDPK() *schema.Table { + return &schema.Table{ + Name: "test_table_success_cq_id", + Resolver: testResolverSuccess, + Columns: []schema.Column{ + schema.CqIDColumn, + { + Name: "test_column", + Type: arrow.PrimitiveTypes.Int64, + PrimaryKey: true, + }, + }, + } +} + +func testTableSuccessWithPKComponents() *schema.Table { + cqID := schema.CqIDColumn + cqID.PrimaryKey = true + return &schema.Table{ + Name: "test_table_succes_vpk__cq_id", + Resolver: testResolverSuccess, + Columns: []schema.Column{ + cqID, + { + Name: "test_column", + Type: arrow.PrimitiveTypes.Int64, + PrimaryKeyComponent: true, + }, + }, + } +} + +func testTableResolverPanic() *schema.Table { + return &schema.Table{ + Name: "test_table_resolver_panic", + Resolver: testResolverPanic, + Columns: []schema.Column{ + { + Name: "test_column", + Type: arrow.PrimitiveTypes.Int64, + }, + }, + } +} + +func testNoTables() *schema.Table { + return nil +} + +func testTablePreResourceResolverPanic() *schema.Table { + return &schema.Table{ + Name: "test_table_pre_resource_resolver_panic", + PreResourceResolver: testPreResourceResolverPanic, + Resolver: testResolverSuccess, + Columns: []schema.Column{ + { + Name: "test_column", + Type: arrow.PrimitiveTypes.Int64, + }, + }, + } +} + +func testTableColumnResolverPanic() *schema.Table { + return &schema.Table{ + Name: "test_table_column_resolver_panic", + Resolver: testResolverSuccess, + Columns: []schema.Column{ + { + Name: "test_column", + Type: arrow.PrimitiveTypes.Int64, + }, + { + Name: "test_column1", + Type: arrow.PrimitiveTypes.Int64, + Resolver: testColumnResolverPanic, + }, + }, + } +} + +func testTableRelationSuccess() *schema.Table { + return &schema.Table{ + Name: "test_table_relation_success", + Resolver: testResolverSuccess, + Columns: []schema.Column{ + { + Name: "test_column", + Type: arrow.PrimitiveTypes.Int64, + }, + }, + Relations: []*schema.Table{ + testTableSuccess(), + }, + } +} + +type syncTestCase struct { + table *schema.Table + data []scalar.Vector + deterministicCQID bool + err error +} + +var syncTestCases = []syncTestCase{ + { + table: testTableSuccess(), + data: []scalar.Vector{ + { + &scalar.Int{Value: 3, Valid: true}, + }, + }, + }, + { + table: testTableResolverPanic(), + data: nil, + }, + { + table: testTablePreResourceResolverPanic(), + data: nil, + }, + + { + table: testNoTables(), + data: nil, + err: ErrNoTables, + }, + + { + table: testTableRelationSuccess(), + data: []scalar.Vector{ + { + &scalar.Int{Value: 3, Valid: true}, + }, + { + &scalar.Int{Value: 3, Valid: true}, + }, + }, + }, + { + table: testTableSuccess(), + data: []scalar.Vector{ + { + &scalar.Int{Value: 3, Valid: true}, + }, + }, + deterministicCQID: true, + }, + { + table: testTableColumnResolverPanic(), + data: []scalar.Vector{ + { + &scalar.Int{Value: 3, Valid: true}, + &scalar.Int{}, + }, + }, + // deterministicCQID: true, + }, + { + table: testTableRelationSuccess(), + data: []scalar.Vector{ + { + &scalar.Int{Value: 3, Valid: true}, + }, + { + &scalar.Int{Value: 3, Valid: true}, + }, + }, + // deterministicCQID: true, + }, + { + table: testTableSuccessWithPK(), + data: []scalar.Vector{ + { + &scalar.Int{Value: 3, Valid: true}, + }, + }, + // deterministicCQID: true, + }, + { + table: testTableSuccessWithCQIDPK(), + data: []scalar.Vector{ + { + // This value will be validated because deterministicCQID is true + &scalar.UUID{Value: [16]byte{194, 83, 85, 170, 181, 44, 91, 112, 164, 224, 201, 153, 31, 90, 59, 135}, Valid: true}, + &scalar.Int{Value: 3, Valid: true}, + }, + }, + deterministicCQID: true, + }, + { + table: testTableSuccessWithCQIDPK(), + data: []scalar.Vector{ + { + // This value will not be validated as it will be randomly set by the scheduler + &scalar.UUID{}, + &scalar.Int{Value: 3, Valid: true}, + }, + }, + deterministicCQID: false, + }, + { + table: testTableSuccessWithPKComponents(), + data: []scalar.Vector{ + { + // This value will not be validated as it will be randomly set by the scheduler + &scalar.UUID{}, + &scalar.Int{Value: 3, Valid: true}, + }, + }, + }, +} + +type optionsTestCase struct { + name string + options []Option +} + +var allOptionsTestCases = []optionsTestCase{ + {name: "default_batching"}, + {name: "without_batching", options: []Option{WithoutBatching()}}, + { + name: "10 rows, 2s", + options: []Option{WithBatchOptions(WithBatchTimeout(2*time.Second), WithBatchMaxRows(10))}, + }, +} + +func TestScheduler(t *testing.T) { + for _, strategy := range AllStrategies { + t.Run(strategy.String(), func(t *testing.T) { + for _, opts := range allOptionsTestCases { + t.Run(opts.name, func(t *testing.T) { + for _, tc := range syncTestCases { + testName := "No table_" + strategy.String() + if tc.table != nil { + tc.table = tc.table.Copy(nil) + testName = tc.table.Name + "_" + strategy.String() + } + t.Run(testName, func(t *testing.T) { + testSyncTable(t, tc, strategy, tc.deterministicCQID, opts.options...) + }) + } + }) + } + }) + } +} + +// nolint:revive +func testSyncTable(t *testing.T, tc syncTestCase, strategy Strategy, deterministicCQID bool, extra ...Option) { + ctx := context.Background() + var tables schema.Tables + if tc.table != nil { + tables = append(tables, tc.table) + } + c := testExecutionClient{} + opts := append([]Option{ + WithLogger(zerolog.New(zerolog.NewTestWriter(t)).Level(zerolog.DebugLevel)), + WithStrategy(strategy), + }, extra...) + sc := NewScheduler(opts...) + msgs := make(chan message.SyncMessage, 10) + resourceCh := make(chan *schema.Resource, 10) + err := sc.Sync(ctx, &c, tables, msgs, resourceCh, WithSyncDeterministicCQID(deterministicCQID)) + require.ErrorIs(t, err, tc.err) + close(msgs) + close(resourceCh) + + var i int + for msg := range msgs { + switch v := msg.(type) { + case *message.SyncInsert: + record := v.Record + rec := tc.data[i].ToArrowRecord(record.Schema()) + if !array.RecordEqual(rec, record) { + // For records that include CqIDColumn, we can't verify equality because it is generated by the scheduler, unless deterministicCQID is true + onlyCqIDInequality := false + for col := range rec.Columns() { + if !deterministicCQID && rec.ColumnName(col) == schema.CqIDColumn.Name { + onlyCqIDInequality = true + continue + } + lc := rec.Column(col) + rc := record.Column(col) + if !array.Equal(lc, rc) { + onlyCqIDInequality = false + } + } + if !onlyCqIDInequality { + t.Fatalf("expected at i=%d: %v. got %v", i, tc.data[i], record) + } + } + i++ + case *message.SyncMigrateTable: + migratedTable := v.Table + + initialTable := tables.Get(v.Table.Name) + + pks := migratedTable.PrimaryKeys() + if (deterministicCQID || len(migratedTable.PrimaryKeyComponents()) > 0) && initialTable.Columns.Get(schema.CqIDColumn.Name) != nil { + if len(pks) != 1 { + t.Fatalf("expected 1 pk. got %d", len(pks)) + } + if pks[0] != schema.CqIDColumn.Name { + t.Fatalf("expected pk name %s. got %s", schema.CqIDColumn.Name, pks[0]) + } + } else if len(pks) != len(initialTable.PrimaryKeys()) { + t.Fatalf("expected 0 pk. got %d", len(pks)) + } + + if len(pks) == 0 { + continue + } + default: + t.Fatalf("expected insert message. got %T", msg) + } + } + if len(tc.data) != i { + t.Fatalf("expected %d resources. got %d", len(tc.data), i) + } +} + +func TestScheduler_Cancellation(t *testing.T) { + data := make([]any, 100) + + tests := []struct { + name string + data []any + cancel bool + messagesOrRows int + }{ + { + name: "should consume all message", + data: data, + cancel: false, + messagesOrRows: len(data) + 1, // 9 data + 1 migration message + }, + { + name: "should not consume all message on cancel", + data: data, + cancel: true, + messagesOrRows: len(data) + 1, // 9 data + 1 migration message + }, + } + + for _, strategy := range AllStrategies { + strategy := strategy + for _, tc := range tests { + tc := tc + t.Run(fmt.Sprintf("%s_%s", tc.name, strategy.String()), func(t *testing.T) { + logger := zerolog.New(zerolog.NewTestWriter(t)) + if tc.cancel { + logger = zerolog.Nop() // FIXME without this, zerolog usage causes a race condition when tests are run with `-race -count=100` + } + sc := NewScheduler(WithLogger(logger), WithStrategy(strategy)) + + messages := make(chan message.SyncMessage) + resourceCh := make(chan *schema.Resource) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go func() { + err := sc.Sync( + ctx, + &testExecutionClient{}, + []*schema.Table{testTableSuccessWithData(tc.data)}, + messages, + resourceCh, + ) + if tc.cancel { + assert.Equal(t, err, context.Canceled) + } else { + require.NoError(t, err) + } + close(messages) + close(resourceCh) + }() + + messagesOrRows := 0 + for msg := range messages { + if tc.cancel { + cancel() + } + if r, ok := msg.(*message.SyncInsert); ok { + messagesOrRows += int(r.Record.NumRows()) + } else { + messagesOrRows++ + } + } + + if tc.cancel { + assert.NotEqual(t, tc.messagesOrRows, messagesOrRows) + } else { + assert.Equal(t, tc.messagesOrRows, messagesOrRows) + } + }) + } + } +} diff --git a/plugin-sdk/scheduler/strategy.go b/plugin-sdk/scheduler/strategy.go new file mode 100644 index 00000000..6464f286 --- /dev/null +++ b/plugin-sdk/scheduler/strategy.go @@ -0,0 +1,101 @@ +package scheduler + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + + "github.com/invopop/jsonschema" +) + +type Strategy int + +func (s *Strategy) String() string { + if s == nil { + return "" + } + return AllStrategyNames[*s] +} + +// MarshalJSON implements json.Marshaler. +func (s *Strategy) MarshalJSON() ([]byte, error) { + var b bytes.Buffer + if s == nil { + b.Write([]byte("null")) + return b.Bytes(), nil + } + b.Write([]byte{'"'}) + b.Write([]byte(s.String())) + b.Write([]byte{'"'}) + return b.Bytes(), nil +} + +// UnmarshalJSON implements json.Unmarshaler. +func (s *Strategy) UnmarshalJSON(b []byte) error { + var name string + if err := json.Unmarshal(b, &name); err != nil { + return err + } + strategy, err := StrategyForName(name) + if err != nil { + return err + } + *s = strategy + return nil +} + +func (s *Strategy) Validate() error { + if s == nil { + return errors.New("scheduler strategy is nil") + } + for _, strategy := range AllStrategies { + if strategy == *s { + return nil + } + } + return fmt.Errorf("unknown scheduler strategy: %d", s) +} + +func (Strategy) JSONSchema() *jsonschema.Schema { + enum := make([]any, len(AllStrategyNames)) + for i, s := range AllStrategyNames { + enum[i] = s + } + return &jsonschema.Schema{ + Type: "string", + Enum: enum, + Default: AllStrategyNames[StrategyDFS], + Title: "CloudQuery scheduling strategy", + } +} + +var AllStrategies = Strategies{StrategyDFS, StrategyRoundRobin, StrategyShuffle} + +var AllStrategyNames = [...]string{ + StrategyDFS: "dfs", + StrategyRoundRobin: "round-robin", + StrategyShuffle: "shuffle", +} + +func StrategyForName(s string) (Strategy, error) { + for i, name := range AllStrategyNames { + if name == s { + return AllStrategies[i], nil + } + } + return StrategyDFS, fmt.Errorf("unknown scheduler strategy: %s", s) +} + +type Strategies []Strategy + +func (s Strategies) String() string { + var buffer bytes.Buffer + for i, strategy := range s { + if i > 0 { + buffer.WriteString(", ") + } + buffer.WriteString(strategy.String()) + } + return buffer.String() +} diff --git a/plugin-sdk/scheduler/strategy_test.go b/plugin-sdk/scheduler/strategy_test.go new file mode 100644 index 00000000..e85d0874 --- /dev/null +++ b/plugin-sdk/scheduler/strategy_test.go @@ -0,0 +1,74 @@ +package scheduler_test + +import ( + _ "embed" + "encoding/json" + "testing" + + "github.com/invopop/jsonschema" + "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/plugin" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/scheduler" +) + +func TestStrategy_JSONSchema(t *testing.T) { + sc := (&jsonschema.Reflector{RequiredFromJSONSchemaTags: true}).Reflect(scheduler.StrategyDFS) + schema, err := json.MarshalIndent(sc, "", " ") + require.NoError(t, err) + + validator, err := plugin.JSONSchemaValidator(string(schema)) + require.NoError(t, err) + + type testCase struct { + Name string + Spec string + Err bool + } + + for _, tc := range []testCase{ + { + Name: "dfs scheduler", + Spec: `"dfs"`, + }, + { + Name: "round-robin scheduler", + Spec: `"round-robin"`, + }, + { + Name: "shuffle scheduler", + Spec: `"shuffle"`, + }, + { + Name: "empty scheduler", + Err: true, + Spec: `""`, + }, + { + Name: "bad scheduler", + Err: true, + Spec: `"bad"`, + }, + { + Name: "bad scheduler type", + Err: true, + Spec: `123`, + }, + { + Name: "null scheduler type", + Err: true, + Spec: `null`, + }, + } { + t.Run(tc.Name, func(t *testing.T) { + var val any + err := json.Unmarshal([]byte(tc.Spec), &val) + require.NoError(t, err) + if tc.Err { + require.Error(t, validator.Validate(val)) + } else { + require.NoError(t, validator.Validate(val)) + } + }) + } +} diff --git a/plugin-sdk/schema/arrow.go b/plugin-sdk/schema/arrow.go new file mode 100644 index 00000000..7d4a0a11 --- /dev/null +++ b/plugin-sdk/schema/arrow.go @@ -0,0 +1,40 @@ +package schema + +import ( + "github.com/apache/arrow/go/v16/arrow" +) + +const ( + MetadataUnique = "cq:extension:unique" + MetadataPrimaryKey = "cq:extension:primary_key" + MetadataPrimaryKeyComponent = "cq:extension:primary_key_component" + MetadataConstraintName = "cq:extension:constraint_name" + MetadataIncremental = "cq:extension:incremental" + + MetadataTrue = "true" + MetadataFalse = "false" + MetadataTableName = "cq:table_name" + MetadataTableDescription = "cq:table_description" + MetadataTableTitle = "cq:table_title" + MetadataTableDependsOn = "cq:table_depends_on" + MetadataTableIsPaid = "cq:table_paid" +) + +type Schemas []*arrow.Schema + +func (s Schemas) Len() int { + return len(s) +} + +func (s Schemas) SchemaByName(name string) *arrow.Schema { + for _, sc := range s { + tableName, ok := sc.Metadata().GetValue(MetadataTableName) + if !ok { + continue + } + if tableName == name { + return sc + } + } + return nil +} diff --git a/plugin-sdk/schema/arrow_test.go b/plugin-sdk/schema/arrow_test.go new file mode 100644 index 00000000..91b1d1e3 --- /dev/null +++ b/plugin-sdk/schema/arrow_test.go @@ -0,0 +1,33 @@ +package schema + +import ( + "fmt" + "strings" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" +) + +func RecordDiff(l arrow.Record, r arrow.Record) string { + var sb strings.Builder + if l.NumCols() != r.NumCols() { + return fmt.Sprintf("different number of columns: %d vs %d", l.NumCols(), r.NumCols()) + } + if l.NumRows() != r.NumRows() { + return fmt.Sprintf("different number of rows: %d vs %d", l.NumRows(), r.NumRows()) + } + for i := 0; i < int(l.NumCols()); i++ { + edits, err := array.Diff(l.Column(i), r.Column(i)) + if err != nil { + panic(fmt.Sprintf("left: %v, right: %v, error: %v", l.Column(i).DataType(), r.Column(i).DataType(), err)) + } + diff := edits.UnifiedDiff(l.Column(i), r.Column(i)) + if diff != "" { + sb.WriteString(l.Schema().Field(i).Name) + sb.WriteString(": ") + sb.WriteString(diff) + sb.WriteString("\n") + } + } + return sb.String() +} diff --git a/plugin-sdk/schema/column.go b/plugin-sdk/schema/column.go new file mode 100644 index 00000000..487b0501 --- /dev/null +++ b/plugin-sdk/schema/column.go @@ -0,0 +1,189 @@ +package schema + +import ( + "context" + "encoding/json" + "strings" + + "github.com/apache/arrow/go/v16/arrow" +) + +type ColumnList []Column + +// ColumnResolver is called for each row received in TableResolver's data fetch. +// execution holds all relevant information regarding execution as well as the Column called. +// resource holds the current row we are resolving the column for. +type ColumnResolver func(ctx context.Context, meta ClientMeta, resource *Resource, c Column) error + +// Column definition for Table +type Column struct { + // Name of column + Name string `json:"name"` + // Value Type of column i.e String, UUID etc' + Type arrow.DataType `json:"type"` + // Description about column, this description is added as a comment in the database + Description string `json:"description"` + // Column Resolver allows to set your own data for a column; this can be an API call, setting multiple embedded values, etc + Resolver ColumnResolver `json:"-"` + + // IgnoreInTests is used to skip verifying the column is non-nil in integration tests. + // By default, integration tests perform a fetch for all resources in cloudquery's test account, and + // verify all columns are non-nil. + // If IgnoreInTests is true, verification is skipped for this column. + // Used when it is hard to create a reproducible environment with this column being non-nil (e.g. various error columns). + IgnoreInTests bool `json:"-"` + + // PrimaryKey requires the destinations supporting this to include this column into the primary key + PrimaryKey bool `json:"primary_key"` + // NotNull requires the destinations supporting this to mark this column as non-nullable + NotNull bool `json:"not_null"` + // IncrementalKey is a flag that indicates if the column is used as part of an incremental key. + // It is mainly used for documentation purposes, but may also be used as part of ensuring that + // migrations are done correctly. + IncrementalKey bool `json:"incremental_key"` + // Unique requires the destinations supporting this to mark this column as unique + Unique bool `json:"unique"` + + // PrimaryKeyComponent is a flag that indicates if the column is used as part of the input to calculate the value of `_cq_id`. + PrimaryKeyComponent bool `json:"primary_key_component"` +} + +// NewColumnFromArrowField creates a new Column from an arrow.Field +// arrow.Field is a low-level representation of a CloudQuery column +// that can be sent over the wire in a cross-language way. +func NewColumnFromArrowField(f arrow.Field) Column { + column := Column{ + Name: f.Name, + Type: f.Type, + NotNull: !f.Nullable, + } + + v, ok := f.Metadata.GetValue(MetadataPrimaryKey) + column.PrimaryKey = ok && v == MetadataTrue + + v, ok = f.Metadata.GetValue(MetadataUnique) + column.Unique = ok && v == MetadataTrue + + v, ok = f.Metadata.GetValue(MetadataIncremental) + column.IncrementalKey = ok && v == MetadataTrue + + v, ok = f.Metadata.GetValue(MetadataPrimaryKeyComponent) + column.PrimaryKeyComponent = ok && v == MetadataTrue + + return column +} + +func (c Column) ToArrowField() arrow.Field { + mdKV := map[string]string{ + MetadataPrimaryKey: MetadataFalse, + MetadataUnique: MetadataFalse, + MetadataIncremental: MetadataFalse, + MetadataPrimaryKeyComponent: MetadataFalse, + } + if c.PrimaryKey { + mdKV[MetadataPrimaryKey] = MetadataTrue + } + if c.Unique { + mdKV[MetadataUnique] = MetadataTrue + } + if c.IncrementalKey { + mdKV[MetadataIncremental] = MetadataTrue + } + if c.PrimaryKeyComponent { + mdKV[MetadataPrimaryKeyComponent] = MetadataTrue + } + + return arrow.Field{ + Name: c.Name, + Type: c.Type, + Nullable: !c.NotNull, + Metadata: arrow.MetadataFrom(mdKV), + } +} + +func (c Column) MarshalJSON() ([]byte, error) { + type Alias struct { + Name string `json:"name"` + Type string `json:"type"` + Description string `json:"description"` + PrimaryKey bool `json:"primary_key"` + NotNull bool `json:"not_null"` + Unique bool `json:"unique"` + IncrementalKey bool `json:"incremental_key"` + PrimaryKeyComponent bool `json:"primary_key_component"` + } + var alias Alias + alias.Name = c.Name + alias.Type = c.Type.String() + alias.Description = c.Description + alias.PrimaryKey = c.PrimaryKey + alias.NotNull = c.NotNull + alias.Unique = c.Unique + alias.IncrementalKey = c.IncrementalKey + alias.PrimaryKeyComponent = c.PrimaryKeyComponent + + return json.Marshal(alias) +} + +func (c Column) String() string { + var sb strings.Builder + sb.WriteString(c.Name) + sb.WriteString(":") + sb.WriteString(c.Type.String()) + if c.PrimaryKey { + sb.WriteString(":PK") + } + if c.NotNull { + sb.WriteString(":NotNull") + } + if c.Unique { + sb.WriteString(":Unique") + } + if c.IncrementalKey { + sb.WriteString(":IncrementalKey") + } + + if c.PrimaryKeyComponent { + sb.WriteString(":PrimaryKeyComponent") + } + return sb.String() +} + +func (c ColumnList) Index(col string) int { + for i, c := range c { + if c.Name == col { + return i + } + } + return -1 +} + +func (c ColumnList) Names() []string { + ret := make([]string, len(c)) + for i := range c { + ret[i] = c[i].Name + } + return ret +} + +func (c ColumnList) Get(name string) *Column { + for i := range c { + if c[i].Name == name { + return &c[i] + } + } + return nil +} + +func (c ColumnList) String() string { + var sb strings.Builder + sb.WriteString("[") + for i, col := range c { + sb.WriteString(col.String()) + if i != len(c)-1 { + sb.WriteString(", ") + } + } + sb.WriteString("]") + return sb.String() +} diff --git a/plugin-sdk/schema/convert.go b/plugin-sdk/schema/convert.go new file mode 100644 index 00000000..2242ea10 --- /dev/null +++ b/plugin-sdk/schema/convert.go @@ -0,0 +1,202 @@ +//nolint:revive,gocritic,unused +package schema + +import ( + "reflect" + "time" +) + +// underlyingNumberType gets the underlying type that can be converted to Int2, Int4, Int8, Float4, or Float8 +func underlyingNumberType(val any) (any, bool) { + refVal := reflect.ValueOf(val) + + switch refVal.Kind() { + case reflect.Ptr: + if refVal.IsNil() { + return nil, false + } + convVal := refVal.Elem().Interface() + return convVal, true + case reflect.Int: + convVal := int(refVal.Int()) + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Int8: + convVal := int8(refVal.Int()) + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Int16: + convVal := int16(refVal.Int()) + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Int32: + convVal := int32(refVal.Int()) + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Int64: + convVal := refVal.Int() + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Uint: + convVal := uint(refVal.Uint()) + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Uint8: + convVal := uint8(refVal.Uint()) + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Uint16: + convVal := uint16(refVal.Uint()) + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Uint32: + convVal := uint32(refVal.Uint()) + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Uint64: + convVal := refVal.Uint() + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Float32: + convVal := float32(refVal.Float()) + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Float64: + convVal := refVal.Float() + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.String: + convVal := refVal.String() + return convVal, reflect.TypeOf(convVal) != refVal.Type() + } + + return nil, false +} + +// underlyingBoolType gets the underlying type that can be converted to Bool +func underlyingBoolType(val any) (any, bool) { + refVal := reflect.ValueOf(val) + + switch refVal.Kind() { + case reflect.Ptr: + if refVal.IsNil() { + return nil, false + } + convVal := refVal.Elem().Interface() + return convVal, true + case reflect.Bool: + convVal := refVal.Bool() + return convVal, reflect.TypeOf(convVal) != refVal.Type() + } + + return nil, false +} + +// underlyingBytesType gets the underlying type that can be converted to []byte +func underlyingBytesType(val any) (any, bool) { + refVal := reflect.ValueOf(val) + + switch refVal.Kind() { + case reflect.Ptr: + if refVal.IsNil() { + return nil, false + } + convVal := refVal.Elem().Interface() + return convVal, true + case reflect.Slice: + if refVal.Type().Elem().Kind() == reflect.Uint8 { + convVal := refVal.Bytes() + return convVal, reflect.TypeOf(convVal) != refVal.Type() + } + } + + return nil, false +} + +// underlyingTimeType gets the underlying type that can be converted to time.Time +func underlyingTimeType(val any) (any, bool) { + refVal := reflect.ValueOf(val) + + switch refVal.Kind() { + case reflect.Ptr: + if refVal.IsNil() { + return nil, false + } + convVal := refVal.Elem().Interface() + return convVal, true + } + + timeType := reflect.TypeOf(time.Time{}) + if refVal.Type().ConvertibleTo(timeType) { + return refVal.Convert(timeType).Interface(), true + } + + return nil, false +} + +// underlyingStringType gets the underlying type that can be converted to String +func underlyingStringType(val any) (any, bool) { + refVal := reflect.ValueOf(val) + + switch refVal.Kind() { + case reflect.Ptr: + if refVal.IsNil() { + return nil, false + } + convVal := refVal.Elem().Interface() + return convVal, true + case reflect.String: + convVal := refVal.String() + return convVal, reflect.TypeOf(convVal) != refVal.Type() + } + + return nil, false +} + +// underlyingUUIDType gets the underlying type that can be converted to [16]byte +func underlyingUUIDType(val any) (any, bool) { + refVal := reflect.ValueOf(val) + + switch refVal.Kind() { + case reflect.Ptr: + if refVal.IsNil() { + return nil, false + } + convVal := refVal.Elem().Interface() + return convVal, true + } + + uuidType := reflect.TypeOf([16]byte{}) + if refVal.Type().ConvertibleTo(uuidType) { + return refVal.Convert(uuidType).Interface(), true + } + + return nil, false +} + +// underlyingSliceType gets the underlying slice type +func underlyingSliceType(val any) (any, bool) { + refVal := reflect.ValueOf(val) + + switch refVal.Kind() { + case reflect.Ptr: + if refVal.IsNil() { + return nil, false + } + convVal := refVal.Elem().Interface() + return convVal, true + case reflect.Slice: + baseSliceType := reflect.SliceOf(refVal.Type().Elem()) + if refVal.Type().ConvertibleTo(baseSliceType) { + convVal := refVal.Convert(baseSliceType) + return convVal.Interface(), reflect.TypeOf(convVal.Interface()) != refVal.Type() + } + } + + return nil, false +} + +// underlyingPtrType dereferences a pointer +func underlyingPtrType(val any) (any, bool) { + refVal := reflect.ValueOf(val) + + //nolint:gocritic + switch refVal.Kind() { + case reflect.Ptr: + if refVal.IsNil() { + return nil, false + } + convVal := refVal.Elem().Interface() + return convVal, true + } + + return nil, false +} diff --git a/plugin-sdk/schema/errors.go b/plugin-sdk/schema/errors.go new file mode 100644 index 00000000..f179185c --- /dev/null +++ b/plugin-sdk/schema/errors.go @@ -0,0 +1,44 @@ +//nolint:revive,gocritic,unused +package schema + +import ( + "fmt" + + "github.com/apache/arrow/go/v16/arrow" +) + +const ( + noConversion = "no conversion available" + cannotDecodeString = "cannot decode from string" + cannotFindDimensions = "cannot find dimensions" + notInterface = "not an interface" + expectedElements = "expected %d elements, but got %d instead" + expectedElementsInDimension = "expected %d elements in dimension %d, but got %d instead" + cannotSetIndex = "cannot set index %d" +) + +type ValidationError struct { + Err error + Msg string + Type arrow.DataType + Value any +} + +func (e *ValidationError) Error() string { + if e.Err == nil { + return fmt.Sprintf("cannot set `%s` with value `%v`: %s", e.Type, e.Value, e.Msg) + } + return fmt.Sprintf("cannot set `%s` with value `%v`: %s (%s)", e.Type, e.Value, e.Msg, e.Err) +} + +// this prints the error without the value +func (e *ValidationError) MaskedError() string { + if e.Err == nil { + return fmt.Sprintf("cannot set `%s`: %s", e.Type, e.Msg) + } + return fmt.Sprintf("cannot set `%s`: %s (%s)", e.Type, e.Msg, e.Err) +} + +func (e *ValidationError) Unwrap() error { + return e.Err +} diff --git a/plugin-sdk/schema/meta.go b/plugin-sdk/schema/meta.go new file mode 100644 index 00000000..0851fdca --- /dev/null +++ b/plugin-sdk/schema/meta.go @@ -0,0 +1,61 @@ +package schema + +import ( + "context" + + "github.com/apache/arrow/go/v16/arrow" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/scalar" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" +) + +type ClientMeta interface { + ID() string +} + +// These columns are managed and populated by the source plugins +var CqIDColumn = Column{ + Name: "_cq_id", + Type: types.ExtensionTypes.UUID, + Description: "Internal CQ ID of the row", + NotNull: true, + Unique: true, +} + +var CqParentIDColumn = Column{ + Name: "_cq_parent_id", + Type: types.ExtensionTypes.UUID, + Description: "Internal CQ ID of the parent row", + Resolver: parentCqUUIDResolver(), + IgnoreInTests: true, +} + +// These columns are managed and populated by the destination plugin. +var CqSyncTimeColumn = Column{ + Name: "_cq_sync_time", + Type: arrow.FixedWidthTypes.Timestamp_us, + Description: "Internal CQ row of when sync was started (this will be the same for all rows in a single fetch)", +} + +var CqSourceNameColumn = Column{ + Name: "_cq_source_name", + Type: arrow.BinaryTypes.String, + Description: "Internal CQ row that references the source plugin name data was retrieved", +} + +func parentCqUUIDResolver() ColumnResolver { + return func(_ context.Context, _ ClientMeta, r *Resource, c Column) error { + if r.Parent == nil { + return r.Set(c.Name, nil) + } + parentCqID := r.Parent.Get(CqIDColumn.Name) + if parentCqID == nil { + return r.Set(c.Name, nil) + } + pUUID, ok := parentCqID.(*scalar.UUID) + if !ok { + return r.Set(c.Name, nil) + } + return r.Set(c.Name, pUUID) + } +} diff --git a/plugin-sdk/schema/resolvers.go b/plugin-sdk/schema/resolvers.go new file mode 100644 index 00000000..edbeac14 --- /dev/null +++ b/plugin-sdk/schema/resolvers.go @@ -0,0 +1,26 @@ +package schema + +import ( + "context" + + "github.com/thoas/go-funk" +) + +// PathResolver resolves a field in the Resource.Item +// +// Examples: +// PathResolver("Field") +// PathResolver("InnerStruct.Field") +// PathResolver("InnerStruct.InnerInnerStruct.Field") +func PathResolver(path string) ColumnResolver { + return func(_ context.Context, _ ClientMeta, r *Resource, c Column) error { + return r.Set(c.Name, funk.Get(r.Item, path, funk.WithAllowZero())) + } +} + +// ParentColumnResolver resolves a column from the parent's table data, if name isn't set the column will be set to null +func ParentColumnResolver(name string) ColumnResolver { + return func(_ context.Context, _ ClientMeta, r *Resource, c Column) error { + return r.Set(c.Name, r.Parent.Get(name)) + } +} diff --git a/plugin-sdk/schema/resource.go b/plugin-sdk/schema/resource.go new file mode 100644 index 00000000..57052af7 --- /dev/null +++ b/plugin-sdk/schema/resource.go @@ -0,0 +1,155 @@ +package schema + +import ( + "crypto/sha256" + "fmt" + "hash" + "slices" + + "github.com/google/uuid" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/scalar" +) + +type Resources []*Resource + +// Resource represents a row in it's associated table, it carries a reference to the original item, and automatically +// generates an Id based on Table's Columns. Resource data can be accessed by the Get and Set methods +type Resource struct { + // Original resource item that wa from prior resolve + Item any + // Set if this is an embedded table + Parent *Resource + // internal fields + Table *Table + // This is sorted result data by column name + data scalar.Vector + // bldr array.RecordBuilder +} + +func NewResourceData(t *Table, parent *Resource, item any) *Resource { + r := Resource{ + Item: item, + Parent: parent, + Table: t, + data: make(scalar.Vector, len(t.Columns)), + } + for i := range r.data { + r.data[i] = scalar.NewScalar(t.Columns[i].Type) + } + return &r +} + +func (r *Resource) Get(columnName string) scalar.Scalar { + index := r.Table.Columns.Index(columnName) + if index == -1 { + // we panic because we want to distinguish between code error and api error + // this also saves additional checks in our testing code + panic(columnName + " column not found") + } + return r.data[index] +} + +// Set sets a column with value. This does validation and conversion to +// one of concrete it returns an error just for backward compatibility +// and panics in case it fails +func (r *Resource) Set(columnName string, value any) error { + index := r.Table.Columns.Index(columnName) + if index == -1 { + // we panic because we want to distinguish between code error and api error + // this also saves additional checks in our testing code + panic(columnName + " column not found") + } + if err := r.data[index].Set(value); err != nil { + panic(fmt.Errorf("failed to set column %s: %w", columnName, err)) + } + return nil +} + +// Override original item (this is useful for apis that follow list/details pattern) +func (r *Resource) SetItem(item any) { + r.Item = item +} + +func (r *Resource) GetItem() any { + return r.Item +} + +func (r *Resource) GetValues() scalar.Vector { + return r.data +} + +//nolint:revive +func (r *Resource) CalculateCQID(deterministicCQID bool) error { + // if `PrimaryKeyComponent` is set, we calculate the CQID based on those components + pkComponents := r.Table.PrimaryKeyComponents() + if len(pkComponents) > 0 { + return r.storeCQID(uuid.NewSHA1(uuid.UUID{}, calculateCqIDValue(r, pkComponents).Sum(nil))) + } + + // If deterministicCQID is false, we generate a random CQID + if !deterministicCQID { + return r.storeCQID(uuid.New()) + } + names := r.Table.PrimaryKeys() + // If there are no primary keys or if CQID is the only PK, we generate a random CQID + if len(names) == 0 || (len(names) == 1 && names[0] == CqIDColumn.Name) { + return r.storeCQID(uuid.New()) + } + + return r.storeCQID(uuid.NewSHA1(uuid.UUID{}, calculateCqIDValue(r, names).Sum(nil))) +} + +func calculateCqIDValue(r *Resource, cols []string) hash.Hash { + h := sha256.New() + slices.Sort(cols) + for _, col := range cols { + // We need to include the column name in the hash because the same value can be present in multiple columns and therefore lead to the same hash + h.Write([]byte(col)) + h.Write([]byte(r.Get(col).String())) + } + return h +} + +func (r *Resource) storeCQID(value uuid.UUID) error { + // We skip if _cq_id is not present. + // Mostly the problem here is because the transformation step is baked into the resolving step + if r.Table.Columns.Get(CqIDColumn.Name) == nil { + return nil + } + b, err := value.MarshalBinary() + if err != nil { + return err + } + return r.Set(CqIDColumn.Name, b) +} + +// Validates that all primary keys have values. +func (r *Resource) Validate() error { + var missingPks []string + for i, c := range r.Table.Columns { + if c.PrimaryKey { + if !r.data[i].IsValid() { + missingPks = append(missingPks, c.Name) + } + } + } + if len(missingPks) > 0 { + return fmt.Errorf("missing primary key on columns: %v", missingPks) + } + return nil +} + +func (rr Resources) TableName() string { + if len(rr) == 0 { + return "" + } + return rr[0].Table.Name +} + +func (rr Resources) ColumnNames() []string { + if len(rr) == 0 { + return []string{} + } + return rr[0].Table.Columns.Names() +} diff --git a/plugin-sdk/schema/table.go b/plugin-sdk/schema/table.go new file mode 100644 index 00000000..b565d522 --- /dev/null +++ b/plugin-sdk/schema/table.go @@ -0,0 +1,627 @@ +package schema + +import ( + "context" + "fmt" + "regexp" + "slices" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/thoas/go-funk" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/glob" +) + +// TableResolver is the main entry point when a table is sync is called. +// +// Table resolver has 3 main arguments: +// - meta(ClientMeta): is the client returned by the plugin.Provider Configure call +// - parent(Resource): resource is the parent resource in case this table is called via parent table (i.e. relation) +// - res(chan any): is a channel to pass results fetched by the TableResolver +type TableResolver func(ctx context.Context, meta ClientMeta, parent *Resource, res chan<- any) error + +type RowResolver func(ctx context.Context, meta ClientMeta, resource *Resource) error + +type Multiplexer func(meta ClientMeta) []ClientMeta + +type Transform func(table *Table) error + +type Tables []*Table + +// Deprecated: SyncSummary is deprecated. +type SyncSummary struct { + Resources uint64 + Errors uint64 + Panics uint64 +} + +type TableColumnChangeType int + +const ( + TableColumnChangeTypeUnknown TableColumnChangeType = iota + TableColumnChangeTypeAdd + TableColumnChangeTypeUpdate + TableColumnChangeTypeRemove + // These are special cases to help users migrate + // As we remove unique constraints on _cq_id columns this will give destination plugins the ability to auto migrate + TableColumnChangeTypeRemoveUniqueConstraint + // Moving from composite pks to singular PK on _cq_id this will give destination plugins the ability to auto migrate + TableColumnChangeTypeMoveToCQOnly +) + +type TableColumnChange struct { + Type TableColumnChangeType + ColumnName string + Current Column + Previous Column +} + +type Table struct { + // Name of table + Name string `json:"name"` + // Title to be used in documentation (optional: will be generated from name if not set) + Title string `json:"title"` + // table description + Description string `json:"description"` + // Columns are the set of fields that are part of this table + Columns ColumnList `json:"columns"` + // Relations are a set of related tables defines + Relations Tables `json:"relations"` + // Transform + Transform Transform `json:"-"` + // Resolver is the main entry point to fetching table data and + Resolver TableResolver `json:"-"` + // Multiplex returns re-purposed meta clients. The sdk will execute the table with each of them + Multiplex Multiplexer `json:"-"` + // PostResourceResolver is called after all columns have been resolved, but before the Resource is sent to be inserted. The ordering of resolvers is: + // (Table) Resolver → PreResourceResolver → ColumnResolvers → PostResourceResolver + PostResourceResolver RowResolver `json:"-"` + // PreResourceResolver is called before all columns are resolved but after Resource is created. The ordering of resolvers is: + // (Table) Resolver → PreResourceResolver → ColumnResolvers → PostResourceResolver + PreResourceResolver RowResolver `json:"-"` + // IsIncremental is a flag that indicates if the table is incremental or not. This flag mainly affects how the table is + // documented. + IsIncremental bool `json:"is_incremental"` + + // IgnoreInTests is used to exclude a table from integration tests. + // By default, integration tests fetch all resources from cloudquery's test account, and verify all tables + // have at least one row. + // When IgnoreInTests is true, integration tests won't fetch from this table. + // Used when it is hard to create a reproducible environment with a row in this table. + IgnoreInTests bool `json:"-"` + + // Parent is the parent table in case this table is called via parent table (i.e. relation) + Parent *Table `json:"-"` + + PkConstraintName string `json:"pk_constraint_name"` + + // IsPaid indicates whether this table is a paid (premium) table. + // This relates to the CloudQuery plugin itself, and should not be confused + // with whether the table makes use of a paid API or not. + IsPaid bool `json:"is_paid"` +} + +var ( + reValidColumnName = regexp.MustCompile(`^[a-z_][a-z\d_]*$`) +) + +// AddCqIDs adds the cq_id and cq_parent_id columns to the table and all its relations +// set cq_id as primary key if no other primary keys +func AddCqIDs(table *Table) { + havePks := len(table.PrimaryKeys()) > 0 + cqIDColumn := CqIDColumn + if !havePks { + cqIDColumn.PrimaryKey = true + } + table.Columns = append( + ColumnList{ + cqIDColumn, + CqParentIDColumn, + }, + table.Columns..., + ) + for _, rel := range table.Relations { + AddCqIDs(rel) + } +} + +// CqIDAsPK sets the cq_id column as primary key if it exists +// and removes the primary key from all other columns +func CqIDAsPK(t *Table) { + cqIDCol := t.Columns.Get(CqIDColumn.Name) + if cqIDCol == nil { + return + } + for i, c := range t.Columns { + if c.Name == CqIDColumn.Name { + // Ensure that the cq_id column is the primary key + t.Columns[i].PrimaryKey = true + continue + } + if !c.PrimaryKey { + continue + } + t.Columns[i].PrimaryKey = false + } +} + +func NewTablesFromArrowSchemas(schemas []*arrow.Schema) (Tables, error) { + tables := make(Tables, len(schemas)) + for i, schema := range schemas { + table, err := NewTableFromArrowSchema(schema) + if err != nil { + return nil, err + } + tables[i] = table + } + return tables, nil +} + +// NewTableFromArrowSchema creates a CloudQuery Table abstraction from an Arrow schema. +// The Arrow schema is a low level representation of a table that can be sent +// over the wire in a cross-language way. +func NewTableFromArrowSchema(sc *arrow.Schema) (*Table, error) { + tableMD := sc.Metadata() + name, found := tableMD.GetValue(MetadataTableName) + if !found { + return nil, fmt.Errorf("missing table name") + } + description, _ := tableMD.GetValue(MetadataTableDescription) + constraintName, _ := tableMD.GetValue(MetadataConstraintName) + title, _ := tableMD.GetValue(MetadataTableTitle) + dependsOn, _ := tableMD.GetValue(MetadataTableDependsOn) + var parent *Table + if dependsOn != "" { + parent = &Table{Name: dependsOn} + } + fields := sc.Fields() + columns := make(ColumnList, len(fields)) + for i, field := range fields { + columns[i] = NewColumnFromArrowField(field) + } + table := &Table{ + Name: name, + Description: description, + PkConstraintName: constraintName, + Columns: columns, + Title: title, + Parent: parent, + } + if isIncremental, found := tableMD.GetValue(MetadataIncremental); found { + table.IsIncremental = isIncremental == MetadataTrue + } + if isPaid, found := tableMD.GetValue(MetadataTableIsPaid); found { + table.IsPaid = isPaid == MetadataTrue + } + return table, nil +} + +func (t TableColumnChangeType) String() string { + switch t { + case TableColumnChangeTypeAdd: + return "add" + case TableColumnChangeTypeUpdate: + return "update" + case TableColumnChangeTypeRemove: + return "remove" + case TableColumnChangeTypeRemoveUniqueConstraint: + return "remove_unique_constraint" + case TableColumnChangeTypeMoveToCQOnly: + return "move_to_cq_only" + default: + return "unknown" + } +} + +func (t TableColumnChange) String() string { + switch t.Type { + case TableColumnChangeTypeAdd: + return fmt.Sprintf("column: %s, type: %s, current: %s", t.ColumnName, t.Type, t.Current) + case TableColumnChangeTypeUpdate: + return fmt.Sprintf("column: %s, type: %s, current: %s, previous: %s", t.ColumnName, t.Type, t.Current, t.Previous) + case TableColumnChangeTypeRemove: + return fmt.Sprintf("column: %s, type: %s, previous: %s", t.ColumnName, t.Type, t.Previous) + case TableColumnChangeTypeRemoveUniqueConstraint: + return fmt.Sprintf("column: %s, previous: %s", t.ColumnName, t.Previous) + case TableColumnChangeTypeMoveToCQOnly: + return fmt.Sprintf("multi-column: %s, type: %s, previous: %s", t.ColumnName, t.Type, t.Previous) + default: + return fmt.Sprintf("column: %s, type: %s, current: %s, previous: %s", t.ColumnName, t.Type, t.Current, t.Previous) + } +} + +func (tt Tables) FilterDfsFunc(include, exclude func(*Table) bool, skipDependentTables bool) Tables { + filteredTables := make(Tables, 0, len(tt)) + for _, t := range tt { + filteredTable := t.Copy(nil) + filteredTable = filteredTable.filterDfs(false, include, exclude, skipDependentTables) + if filteredTable != nil { + filteredTables = append(filteredTables, filteredTable) + } + } + return filteredTables +} + +func (tt Tables) ToArrowSchemas() Schemas { + flattened := tt.FlattenTables() + schemas := make(Schemas, len(flattened)) + for i, t := range flattened { + schemas[i] = t.ToArrowSchema() + } + return schemas +} + +func (tt Tables) FilterDfs(tables, skipTables []string, skipDependentTables bool) (Tables, error) { + flattenedTables := tt.FlattenTables() + for _, includePattern := range tables { + matched := false + for _, table := range flattenedTables { + if glob.Glob(includePattern, table.Name) { + matched = true + break + } + } + if !matched { + return nil, fmt.Errorf("tables include a pattern %s with no matches", includePattern) + } + } + for _, excludePattern := range skipTables { + matched := false + for _, table := range flattenedTables { + if glob.Glob(excludePattern, table.Name) { + matched = true + break + } + } + if !matched { + return nil, fmt.Errorf("skip_tables include a pattern %s with no matches", excludePattern) + } + } + include := func(t *Table) bool { + for _, includePattern := range tables { + if glob.Glob(includePattern, t.Name) { + return true + } + } + return false + } + exclude := func(t *Table) bool { + for _, skipPattern := range skipTables { + if glob.Glob(skipPattern, t.Name) { + return true + } + } + return false + } + return tt.FilterDfsFunc(include, exclude, skipDependentTables), nil +} + +func (tt Tables) flattenTablesRecursive() Tables { + tables := make(Tables, 0, len(tt)) + for _, t := range tt { + table := *t + table.Relations = nil + tables = append(tables, &table) + tables = append(tables, t.Relations.flattenTablesRecursive()...) + } + + return tables +} + +func (tt Tables) FlattenTables() Tables { + tables := tt.flattenTablesRecursive() + + seen := make(map[string]struct{}) + deduped := make(Tables, 0, len(tables)) + for _, t := range tables { + if _, found := seen[t.Name]; !found { + deduped = append(deduped, t) + seen[t.Name] = struct{}{} + } + } + + return slices.Clip(deduped) +} + +// UnflattenTables returns a new Tables copy with the relations unflattened. This is the +// opposite operation of FlattenTables. +func (tt Tables) UnflattenTables() (Tables, error) { + tables := make(Tables, 0, len(tt)) + for _, t := range tt { + table := *t + tables = append(tables, &table) + } + topLevel := make([]*Table, 0, len(tt)) + // build relations + for _, table := range tables { + if table.Parent == nil { + topLevel = append(topLevel, table) + continue + } + parent := tables.Get(table.Parent.Name) + if parent == nil { + return nil, fmt.Errorf("parent table %s not found", table.Parent.Name) + } + table.Parent = parent + parent.Relations = append(parent.Relations, table) + } + return slices.Clip(topLevel), nil +} + +func (tt Tables) TableNames() []string { + ret := []string{} + for _, t := range tt { + ret = append(ret, t.TableNames()...) + } + return ret +} + +// GetTopLevel returns a table by name. Only returns the table if it is in top-level list. +func (tt Tables) GetTopLevel(name string) *Table { + for _, t := range tt { + if t.Name == name { + return t + } + } + return nil +} + +// Get returns a table by name. Returns top-level tables and relations. +func (tt Tables) Get(name string) *Table { + for _, t := range tt { + if t.Name == name { + return t + } + table := t.Relations.Get(name) + if table != nil { + return table + } + } + return nil +} + +func (tt Tables) ValidateDuplicateColumns() error { + for _, t := range tt { + if err := t.ValidateDuplicateColumns(); err != nil { + return err + } + } + return nil +} + +func (tt Tables) ValidateDuplicateTables() error { + tableNames := tt.TableNames() + tables := make(map[string]bool, len(tt)) + for _, t := range tableNames { + if _, ok := tables[t]; ok { + return fmt.Errorf("duplicate table %s", t) + } + tables[t] = true + } + return nil +} + +func (tt Tables) GetPaidTables() Tables { + flattenedTables := tt.FlattenTables() + paidTables := make(Tables, 0, len(flattenedTables)) + for i := range flattenedTables { + if flattenedTables[i].IsPaid { + paidTables = append(paidTables, flattenedTables[i]) + } + } + return paidTables +} + +func (tt Tables) HasPaidTables() bool { + return len(tt.GetPaidTables()) > 0 +} + +// this will filter the tree in-place +func (t *Table) filterDfs(parentMatched bool, include, exclude func(*Table) bool, skipDependentTables bool) *Table { + if exclude(t) { + return nil + } + matched := parentMatched && !skipDependentTables + if include(t) { + matched = true + } + filteredRelations := make([]*Table, 0, len(t.Relations)) + childMatched := false + for _, r := range t.Relations { + filteredChild := r.filterDfs(matched, include, exclude, skipDependentTables) + if filteredChild != nil { + childMatched = true + filteredRelations = append(filteredRelations, r) + } + } + t.Relations = filteredRelations + if matched || childMatched { + return t + } + return nil +} + +func (t *Table) PrimaryKeysIndexes() []int { + var primaryKeys []int + for i, c := range t.Columns { + if c.PrimaryKey { + primaryKeys = append(primaryKeys, i) + } + } + + return primaryKeys +} + +func (t *Table) ToArrowSchema() *arrow.Schema { + fields := make([]arrow.Field, len(t.Columns)) + md := map[string]string{ + MetadataTableName: t.Name, + MetadataTableDescription: t.Description, + MetadataTableTitle: t.Title, + MetadataConstraintName: t.PkConstraintName, + } + if t.IsIncremental { + md[MetadataIncremental] = MetadataTrue + } + if t.Parent != nil { + md[MetadataTableDependsOn] = t.Parent.Name + } + if t.IsPaid { + md[MetadataTableIsPaid] = MetadataTrue + } + schemaMd := arrow.MetadataFrom(md) + for i, c := range t.Columns { + fields[i] = c.ToArrowField() + } + return arrow.NewSchema(fields, &schemaMd) +} + +// GetChanges returns changes between two tables when t is the new one and old is the old one. +func (t *Table) GetChanges(old *Table) []TableColumnChange { + var changes []TableColumnChange + + // Special case: Moving from individual pks to singular PK on _cq_id + newPks := t.PrimaryKeys() + if len(newPks) == 1 && newPks[0] == CqIDColumn.Name && !funk.Contains(old.PrimaryKeys(), CqIDColumn.Name) && len(old.PrimaryKeys()) > 0 { + changes = append(changes, TableColumnChange{ + Type: TableColumnChangeTypeMoveToCQOnly, + }) + } + for _, c := range t.Columns { + otherColumn := old.Columns.Get(c.Name) + // A column was added to the table definition + if otherColumn == nil { + changes = append(changes, TableColumnChange{ + Type: TableColumnChangeTypeAdd, + ColumnName: c.Name, + Current: c, + }) + continue + } + + // Column type or options (e.g. PK, Not Null) changed in the new table definition + if !arrow.TypeEqual(c.Type, otherColumn.Type) || c.NotNull != otherColumn.NotNull || c.PrimaryKey != otherColumn.PrimaryKey { + changes = append(changes, TableColumnChange{ + Type: TableColumnChangeTypeUpdate, + ColumnName: c.Name, + Current: c, + Previous: *otherColumn, + }) + } + + // Unique constraint was removed + if !c.Unique && otherColumn.Unique { + changes = append(changes, TableColumnChange{ + Type: TableColumnChangeTypeRemoveUniqueConstraint, + ColumnName: c.Name, + Previous: *otherColumn, + }) + } + } + // A column was removed from the table definition + for _, c := range old.Columns { + if t.Columns.Get(c.Name) == nil { + changes = append(changes, TableColumnChange{ + Type: TableColumnChangeTypeRemove, + ColumnName: c.Name, + Previous: c, + }) + } + } + return changes +} + +func (t *Table) ValidateDuplicateColumns() error { + columns := make(map[string]bool, len(t.Columns)) + for _, c := range t.Columns { + if _, ok := columns[c.Name]; ok { + return fmt.Errorf("duplicate column %s in table %s", c.Name, t.Name) + } + columns[c.Name] = true + } + for _, rel := range t.Relations { + if err := rel.ValidateDuplicateColumns(); err != nil { + return err + } + } + return nil +} + +func (t *Table) Column(name string) *Column { + for _, c := range t.Columns { + if c.Name == name { + return &c + } + } + return nil +} + +// OverwriteOrAddColumn overwrites or adds columns. +// If the column with the same name exists, overwrites it. +// Otherwise, adds the column to the beginning of the table. +func (t *Table) OverwriteOrAddColumn(column *Column) { + for i, c := range t.Columns { + if c.Name == column.Name { + t.Columns[i] = *column + return + } + } + t.Columns = append([]Column{*column}, t.Columns...) +} + +func (t *Table) PrimaryKeys() []string { + var primaryKeys []string + for _, c := range t.Columns { + if c.PrimaryKey { + primaryKeys = append(primaryKeys, c.Name) + } + } + + return primaryKeys +} + +func (t *Table) IncrementalKeys() []string { + var incrementalKeys []string + for _, c := range t.Columns { + if c.IncrementalKey { + incrementalKeys = append(incrementalKeys, c.Name) + } + } + + return incrementalKeys +} + +func (t *Table) PrimaryKeyComponents() []string { + var primaryKeyComponents []string + for _, c := range t.Columns { + if c.PrimaryKeyComponent { + primaryKeyComponents = append(primaryKeyComponents, c.Name) + } + } + + return primaryKeyComponents +} + +func (t *Table) TableNames() []string { + ret := []string{t.Name} + for _, rel := range t.Relations { + ret = append(ret, rel.TableNames()...) + } + return ret +} + +func (t *Table) Copy(parent *Table) *Table { + c := *t + c.Parent = parent + c.Columns = make([]Column, len(t.Columns)) + copy(c.Columns, t.Columns) + c.Relations = make([]*Table, len(t.Relations)) + for i := range t.Relations { + c.Relations[i] = t.Relations[i].Copy(&c) + } + return &c +} + +func ValidColumnName(name string) bool { + return reValidColumnName.MatchString(name) +} diff --git a/plugin-sdk/schema/table_test.go b/plugin-sdk/schema/table_test.go new file mode 100644 index 00000000..4bf39742 --- /dev/null +++ b/plugin-sdk/schema/table_test.go @@ -0,0 +1,767 @@ +package schema + +import ( + "testing" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" +) + +var testTable = &Table{ + Name: "test", + Columns: []Column{}, + Relations: []*Table{ + { + Name: "test2", + Columns: []Column{}, + Parent: &Table{Name: "test"}, + }, + }, +} + +func TestTablesFlatten(t *testing.T) { + srcTables := Tables{testTable} + tables := srcTables.FlattenTables() + require.Equal(t, 1, len(srcTables)) // verify that the source Tables were left untouched + require.Equal(t, 1, len(testTable.Relations)) + require.Equal(t, 2, len(tables)) + for _, table := range tables { + require.Nil(t, table.Relations) + } + + srcTables = Tables{testTable, testTable} + tables = srcTables.FlattenTables() + require.Equal(t, 2, len(srcTables)) // verify that the source Tables were left untouched + require.Equal(t, 1, len(testTable.Relations)) + require.Equal(t, 2, len(tables)) + for _, table := range tables { + require.Nil(t, table.Relations) + } + + tables = tables.FlattenTables() + if len(tables) != 2 { + t.Fatal("expected 2 tables") + } +} + +func TestTablesUnflatten(t *testing.T) { + srcTables := Tables{testTable} + tables, err := srcTables.FlattenTables().UnflattenTables() + require.NoError(t, err) + require.Equal(t, 1, len(srcTables)) // verify that the source Tables were left untouched + require.Equal(t, 1, len(tables)) // verify that the tables are equal to what we started with + require.Equal(t, 1, len(tables[0].Relations)) +} + +func TestTablesFilterDFS(t *testing.T) { + tests := []struct { + name string + tables Tables + configurationTables []string + configurationSkipTables []string + skipDependentTables bool + want []string + err string + }{ + { + name: "should return all tables when '*' is provided", + tables: []*Table{{Name: "table1"}, {Name: "table2"}, {Name: "table3"}}, + configurationTables: []string{"*"}, + want: []string{"table1", "table2", "table3"}, + }, + { + name: "should return all tables when '*' is provided, excluding skipped tables", + tables: []*Table{{Name: "table1"}, {Name: "table2"}, {Name: "table3"}}, + configurationTables: []string{"*"}, + configurationSkipTables: []string{"table1", "table3"}, + want: []string{"table2"}, + }, + { + name: "should return the parent and all its descendants", + tables: []*Table{ + { + Name: "main_table", + Relations: []*Table{ + { + Name: "sub_table", + Relations: []*Table{ + { + Name: "sub_sub_table", + }, + }, + }, + }, + }, + }, + configurationTables: []string{"main_table"}, + configurationSkipTables: []string{}, + want: []string{"main_table", "sub_table", "sub_sub_table"}, + }, + { + name: "should return the parent and all its descendants, but skip children and their descendants if they are skipped", + tables: []*Table{{Name: "main_table", Relations: []*Table{ + {Name: "sub_table", Parent: &Table{Name: "main_table"}, Relations: []*Table{ + {Name: "sub_sub_table", Parent: &Table{Name: "sub_table"}}, + }}}}, + }, + configurationTables: []string{"main_table"}, + configurationSkipTables: []string{"sub_table"}, + want: []string{"main_table"}, + }, + { + name: "should return only parent table if child table is skipped", + tables: []*Table{{Name: "main_table", Relations: []*Table{{Name: "sub_table", Parent: &Table{Name: "main_table"}}}}}, + configurationTables: []string{"*"}, + configurationSkipTables: []string{"sub_table"}, + want: []string{"main_table"}, + }, + { + name: "should return specific tables when they are provided", + tables: []*Table{{Name: "table1"}, {Name: "table2"}, {Name: "table3"}}, + configurationTables: []string{"table1"}, + want: []string{"table1"}, + }, + { + name: "should return tables matching glob pattern", + tables: []*Table{{Name: "table1"}, {Name: "table2"}}, + configurationTables: []string{"table*"}, + configurationSkipTables: []string{"table2"}, + want: []string{"table1"}, + }, + { + name: "should not return an error when included table is skipped", + tables: []*Table{{Name: "table1"}, {Name: "table2"}}, + configurationTables: []string{"table2", "table1"}, + configurationSkipTables: []string{"table1"}, + want: []string{"table2"}, + }, + { + name: "should return both tables if child and parent tables are specified", + tables: []*Table{{Name: "main_table", Relations: []*Table{{Name: "sub_table", Parent: &Table{Name: "main_table"}}}}}, + configurationTables: []string{"main_table", "sub_table"}, + want: []string{"main_table", "sub_table"}, + }, + { + name: "should return table only once, even if it is matched by multiple rules", + tables: []*Table{{Name: "table1"}, {Name: "table2"}}, + configurationTables: []string{"*", "table2", "table1", "table*"}, + configurationSkipTables: []string{"table2"}, + want: []string{"table1"}, + }, + { + name: "should match prefix globs", + tables: []*Table{{Name: "table1"}, {Name: "table2"}}, + configurationTables: []string{"*2"}, + configurationSkipTables: []string{}, + want: []string{"table2"}, + }, + { + name: "should match suffix globs", + tables: []*Table{{Name: "table1"}, {Name: "table2"}}, + configurationTables: []string{"table*"}, + configurationSkipTables: []string{}, + want: []string{"table1", "table2"}, + }, + { + name: "should match middle globs", + tables: []*Table{{Name: "table1"}, {Name: "table2"}}, + configurationTables: []string{"t*b*1"}, + configurationSkipTables: []string{}, + want: []string{"table1"}, + }, + { + name: "should skip globs", + tables: []*Table{{Name: "table1"}, {Name: "table2"}}, + configurationTables: []string{"*"}, + configurationSkipTables: []string{"t*1"}, + want: []string{"table2"}, + }, + { + name: "should skip multiple globs", + tables: []*Table{{Name: "table1"}, {Name: "table2"}, {Name: "table3"}}, + configurationTables: []string{"*"}, + configurationSkipTables: []string{"t*1", "t*2"}, + want: []string{"table3"}, + }, + { + name: "should glob match against child tables", + tables: []*Table{{Name: "main_table", Relations: []*Table{{Name: "sub_table", Parent: &Table{Name: "main_table"}}}}}, + configurationTables: []string{"*"}, + configurationSkipTables: []string{}, + want: []string{"main_table", "sub_table"}, + }, + { + name: "should skip parent table", + tables: []*Table{{Name: "main_table", Relations: []*Table{{Name: "sub_table", Parent: &Table{Name: "main_table"}}}}}, + configurationTables: []string{"*"}, + configurationSkipTables: []string{"main_table"}, + want: []string{}, + }, + { + name: "should skip parent table", + tables: []*Table{{Name: "main_table", Relations: []*Table{{Name: "sub_table", Parent: &Table{Name: "main_table"}}}}}, + configurationTables: []string{"*"}, + configurationSkipTables: []string{"main_table1"}, + want: []string{}, + err: "skip_tables include a pattern main_table1 with no matches", + }, + { + name: "should skip parent table", + tables: []*Table{{Name: "main_table", Relations: []*Table{{Name: "sub_table", Parent: &Table{Name: "main_table"}}}}}, + configurationTables: []string{"main_table1"}, + configurationSkipTables: []string{}, + want: []string{}, + err: "tables include a pattern main_table1 with no matches", + }, + { + name: "skip child table but return siblings", + tables: []*Table{ + {Name: "main_table", Relations: []*Table{ + {Name: "sub_table_1", Parent: &Table{Name: "main_table"}}, + {Name: "sub_table_2", Parent: &Table{Name: "main_table"}}}}}, + configurationTables: []string{"main_table"}, + configurationSkipTables: []string{"sub_table_2"}, + want: []string{"main_table", "sub_table_1"}, + }, + { + name: "when specifying a single child table, return only the parent and the specified child", + tables: []*Table{ + {Name: "main_table", Relations: []*Table{ + {Name: "sub_table_1", Parent: &Table{Name: "main_table"}}, + {Name: "sub_table_2", Parent: &Table{Name: "main_table"}}}}}, + configurationTables: []string{"sub_table_1"}, + want: []string{"main_table", "sub_table_1"}, + }, + { + name: "when specifying a leaf table, return only the parents and the leaf", + tables: []*Table{ + {Name: "0", Relations: []*Table{ + {Name: "0_1", Parent: &Table{Name: "0"}, Relations: []*Table{ + {Name: "0_1_1", Parent: &Table{Name: "0_1"}, Relations: []*Table{ + {Name: "0_1_1_1", Parent: &Table{Name: "0_1_1"}}, + {Name: "0_1_1_2", Parent: &Table{Name: "0_1_1"}, Relations: []*Table{ + {Name: "0_1_1_2_1", Parent: &Table{Name: "0_1_1_2"}}, + {Name: "0_1_1_2_2", Parent: &Table{Name: "0_1_1_2"}}, + }}, + {Name: "0_1_1_3", Parent: &Table{Name: "0_1_1"}}, + }}, + {Name: "0_1_2", Parent: &Table{Name: "0_1"}, Relations: []*Table{ + {Name: "0_1_2_1", Parent: &Table{Name: "0_1_2"}}, + {Name: "0_1_2_2", Parent: &Table{Name: "0_1_2"}}, + {Name: "0_1_2_3", Parent: &Table{Name: "0_1_2"}}, + }}, + {Name: "0_1_3", Parent: &Table{Name: "0_1"}}, + }}, + {Name: "0_2", Parent: &Table{Name: "0"}, Relations: []*Table{ + {Name: "0_2_1", Parent: &Table{Name: "0_2"}}, + {Name: "0_2_2", Parent: &Table{Name: "0_2"}}, + }}, + {Name: "0_3", Parent: &Table{Name: "0"}}, + }}, + {Name: "1", Relations: []*Table{ + {Name: "1_1", Parent: &Table{Name: "1"}, Relations: []*Table{ + {Name: "1_1_1", Parent: &Table{Name: "1_1"}, Relations: []*Table{ + {Name: "1_1_1_1", Parent: &Table{Name: "1_1_1"}}, + {Name: "1_1_1_2", Parent: &Table{Name: "1_1_1"}, Relations: []*Table{ + {Name: "1_1_1_2_1", Parent: &Table{Name: "1_1_1_2"}}, + {Name: "1_1_1_2_2", Parent: &Table{Name: "1_1_1_2"}}, + }}, + {Name: "1_1_1_3", Parent: &Table{Name: "1_1_1"}}, + }}, + {Name: "1_1_2", Parent: &Table{Name: "1_1"}, Relations: []*Table{ + {Name: "1_1_2_1", Parent: &Table{Name: "1_1_2"}}, + {Name: "1_1_2_2", Parent: &Table{Name: "1_1_2"}}, + {Name: "1_1_2_3", Parent: &Table{Name: "1_1_2"}}, + }}, + {Name: "1_1_3", Parent: &Table{Name: "1_1"}}, + }}, + {Name: "1_2", Parent: &Table{Name: "1"}, Relations: []*Table{ + {Name: "1_2_1", Parent: &Table{Name: "1_2"}}, + {Name: "1_2_2", Parent: &Table{Name: "1_2"}}, + }}, + {Name: "1_3", Parent: &Table{Name: "1"}}, + }}, + }, + configurationTables: []string{"0_1_1_2_2", "0_1_2_3", "1_1_2_3"}, + want: []string{"0", "0_1", "0_1_1", "0_1_1_2", "0_1_1_2_2", "0_1_2", "0_1_2_3", "1", "1_1", "1_1_2", "1_1_2_3"}, + }, + { + name: "when specifying a descendant table, return the parents, the specified descendant and all its descendant if skip_dependent_tables is false", + tables: []*Table{ + {Name: "0", Relations: []*Table{ + {Name: "0_1", Parent: &Table{Name: "0"}, Relations: []*Table{ + {Name: "0_1_1", Parent: &Table{Name: "0_1"}, Relations: []*Table{ + {Name: "0_1_1_1", Parent: &Table{Name: "0_1_1"}}, + {Name: "0_1_1_2", Parent: &Table{Name: "0_1_1"}, Relations: []*Table{ + {Name: "0_1_1_2_1", Parent: &Table{Name: "0_1_1_2"}}, + {Name: "0_1_1_2_2", Parent: &Table{Name: "0_1_1_2"}}, + }}, + {Name: "0_1_1_3", Parent: &Table{Name: "0_1_1"}}, + }}, + {Name: "0_1_2", Parent: &Table{Name: "0_1"}, Relations: []*Table{ + {Name: "0_1_2_1", Parent: &Table{Name: "0_1_2"}}, + {Name: "0_1_2_2", Parent: &Table{Name: "0_1_2"}}, + {Name: "0_1_2_3", Parent: &Table{Name: "0_1_2"}}, + }}, + {Name: "0_1_3", Parent: &Table{Name: "0_1"}}, + }}, + {Name: "0_2", Parent: &Table{Name: "0"}, Relations: []*Table{ + {Name: "0_2_1", Parent: &Table{Name: "0_2"}}, + {Name: "0_2_2", Parent: &Table{Name: "0_2"}}, + }}, + {Name: "0_3", Parent: &Table{Name: "0"}}, + }}, + {Name: "1", Relations: []*Table{ + {Name: "1_1", Parent: &Table{Name: "1"}, Relations: []*Table{ + {Name: "1_1_1", Parent: &Table{Name: "1_1"}, Relations: []*Table{ + {Name: "1_1_1_1", Parent: &Table{Name: "1_1_1"}}, + {Name: "1_1_1_2", Parent: &Table{Name: "1_1_1"}, Relations: []*Table{ + {Name: "1_1_1_2_1", Parent: &Table{Name: "1_1_1_2"}}, + {Name: "1_1_1_2_2", Parent: &Table{Name: "1_1_1_2"}}, + }}, + {Name: "1_1_1_3", Parent: &Table{Name: "1_1_1"}}, + }}, + {Name: "1_1_2", Parent: &Table{Name: "1_1"}, Relations: []*Table{ + {Name: "1_1_2_1", Parent: &Table{Name: "1_1_2"}}, + {Name: "1_1_2_2", Parent: &Table{Name: "1_1_2"}}, + {Name: "1_1_2_3", Parent: &Table{Name: "1_1_2"}}, + }}, + {Name: "1_1_3", Parent: &Table{Name: "1_1"}}, + }}, + {Name: "1_2", Parent: &Table{Name: "1"}, Relations: []*Table{ + {Name: "1_2_1", Parent: &Table{Name: "1_2"}}, + {Name: "1_2_2", Parent: &Table{Name: "1_2"}}, + }}, + {Name: "1_3", Parent: &Table{Name: "1"}}, + }}, + }, + configurationTables: []string{"1_1_1_2"}, + want: []string{"1", "1_1", "1_1_1", "1_1_1_2", "1_1_1_2_1", "1_1_1_2_2"}, + }, + { + name: "when specifying a descendant table, return the parents and only the specified descendant if skip_dependent_tables is true", + tables: []*Table{ + {Name: "0", Relations: []*Table{ + {Name: "0_1", Parent: &Table{Name: "0"}, Relations: []*Table{ + {Name: "0_1_1", Parent: &Table{Name: "0_1"}, Relations: []*Table{ + {Name: "0_1_1_1", Parent: &Table{Name: "0_1_1"}}, + {Name: "0_1_1_2", Parent: &Table{Name: "0_1_1"}, Relations: []*Table{ + {Name: "0_1_1_2_1", Parent: &Table{Name: "0_1_1_2"}}, + {Name: "0_1_1_2_2", Parent: &Table{Name: "0_1_1_2"}}, + }}, + {Name: "0_1_1_3", Parent: &Table{Name: "0_1_1"}}, + }}, + {Name: "0_1_2", Parent: &Table{Name: "0_1"}, Relations: []*Table{ + {Name: "0_1_2_1", Parent: &Table{Name: "0_1_2"}}, + {Name: "0_1_2_2", Parent: &Table{Name: "0_1_2"}}, + {Name: "0_1_2_3", Parent: &Table{Name: "0_1_2"}}, + }}, + {Name: "0_1_3", Parent: &Table{Name: "0_1"}}, + }}, + {Name: "0_2", Parent: &Table{Name: "0"}, Relations: []*Table{ + {Name: "0_2_1", Parent: &Table{Name: "0_2"}}, + {Name: "0_2_2", Parent: &Table{Name: "0_2"}}, + }}, + {Name: "0_3", Parent: &Table{Name: "0"}}, + }}, + {Name: "1", Relations: []*Table{ + {Name: "1_1", Parent: &Table{Name: "1"}, Relations: []*Table{ + {Name: "1_1_1", Parent: &Table{Name: "1_1"}, Relations: []*Table{ + {Name: "1_1_1_1", Parent: &Table{Name: "1_1_1"}}, + {Name: "1_1_1_2", Parent: &Table{Name: "1_1_1"}, Relations: []*Table{ + {Name: "1_1_1_2_1", Parent: &Table{Name: "1_1_1_2"}}, + {Name: "1_1_1_2_2", Parent: &Table{Name: "1_1_1_2"}}, + }}, + {Name: "1_1_1_3", Parent: &Table{Name: "1_1_1"}}, + }}, + {Name: "1_1_2", Parent: &Table{Name: "1_1"}, Relations: []*Table{ + {Name: "1_1_2_1", Parent: &Table{Name: "1_1_2"}}, + {Name: "1_1_2_2", Parent: &Table{Name: "1_1_2"}}, + {Name: "1_1_2_3", Parent: &Table{Name: "1_1_2"}}, + }}, + {Name: "1_1_3", Parent: &Table{Name: "1_1"}}, + }}, + {Name: "1_2", Parent: &Table{Name: "1"}, Relations: []*Table{ + {Name: "1_2_1", Parent: &Table{Name: "1_2"}}, + {Name: "1_2_2", Parent: &Table{Name: "1_2"}}, + }}, + {Name: "1_3", Parent: &Table{Name: "1"}}, + }}, + }, + configurationTables: []string{"1_1_1_2"}, + skipDependentTables: true, + want: []string{"1", "1_1", "1_1_1", "1_1_1_2"}, + }, + { + name: "skip child tables if skip_dependent_tables is true", + tables: []*Table{ + {Name: "main_table", Relations: []*Table{ + {Name: "sub_table_1", Parent: &Table{Name: "main_table"}}, + {Name: "sub_table_2", Parent: &Table{Name: "main_table"}}}}}, + configurationTables: []string{"main_table"}, + configurationSkipTables: []string{}, + skipDependentTables: true, + want: []string{"main_table"}, + }, + { + name: "skip child tables if skip_dependent_tables is true, but not if explicitly included", + tables: []*Table{ + {Name: "main_table_1", Relations: []*Table{ + {Name: "sub_table_1"}, + }}, + {Name: "main_table_2", Relations: []*Table{ + {Name: "sub_table_2", Parent: &Table{Name: "main_table"}}, + {Name: "sub_table_3", Parent: &Table{Name: "main_table"}}}}}, + configurationTables: []string{"main_table_1", "sub_table_2"}, + configurationSkipTables: []string{}, + skipDependentTables: true, + want: []string{"main_table_1", "main_table_2", "sub_table_2"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotTables, err := tt.tables.FilterDfs(tt.configurationTables, tt.configurationSkipTables, tt.skipDependentTables) + // nolint:gocritic + if err != nil && tt.err == "" { + t.Errorf("got error %v, want nil", err) + } else if err != nil && tt.err != "" && err.Error() != tt.err { + t.Errorf("got error %v, want %v", err, tt.err) + } else if err == nil && tt.err != "" { + t.Errorf("got nil, want error %v", tt.err) + } + gotTables = gotTables.FlattenTables() + gotNames := make([]string, len(gotTables)) + for i := range gotTables { + gotNames[i] = gotTables[i].Name + } + if diff := cmp.Diff(gotNames, tt.want); diff != "" { + t.Errorf("diff (+got, -want): %v", diff) + } + }) + } +} + +type testTableGetChangeTestCase struct { + name string + target *Table + source *Table + expectedChanges []TableColumnChange +} + +var testTableGetChangeTestCases = []testTableGetChangeTestCase{ + { + name: "no changes", + target: &Table{ + Name: "test", + Columns: []Column{ + {Name: "bool", Type: arrow.FixedWidthTypes.Boolean}, + }, + }, + source: &Table{ + Name: "test", + Columns: []Column{ + {Name: "bool", Type: arrow.FixedWidthTypes.Boolean}, + }, + }, + expectedChanges: nil, + }, + { + name: "add column", + target: &Table{ + Name: "test", + Columns: []Column{ + {Name: "bool", Type: arrow.FixedWidthTypes.Boolean}, + {Name: "bool1", Type: arrow.FixedWidthTypes.Boolean}, + }, + }, + source: &Table{ + Name: "test", + Columns: []Column{ + {Name: "bool", Type: arrow.FixedWidthTypes.Boolean}, + }, + }, + expectedChanges: []TableColumnChange{ + { + Type: TableColumnChangeTypeAdd, + ColumnName: "bool1", + Current: Column{Name: "bool1", Type: arrow.FixedWidthTypes.Boolean}, + }, + }, + }, + { + name: "remove column", + target: &Table{ + Name: "test", + Columns: []Column{ + {Name: "bool", Type: arrow.FixedWidthTypes.Boolean}, + }, + }, + source: &Table{ + Name: "test", + Columns: []Column{ + {Name: "bool", Type: arrow.FixedWidthTypes.Boolean}, + {Name: "bool1", Type: arrow.FixedWidthTypes.Boolean}, + }, + }, + expectedChanges: []TableColumnChange{ + { + Type: TableColumnChangeTypeRemove, + ColumnName: "bool1", + Previous: Column{Name: "bool1", Type: arrow.FixedWidthTypes.Boolean}, + }, + }, + }, + + { + name: "move to cq_id as primary key", + target: &Table{ + Name: "test", + Columns: []Column{ + { + Name: "_cq_id", + Type: types.ExtensionTypes.UUID, + Description: "Internal CQ ID of the row", + NotNull: true, + Unique: true, + PrimaryKey: true, + }, + {Name: "bool", Type: arrow.FixedWidthTypes.Boolean, PrimaryKey: false}, + }, + }, + source: &Table{ + Name: "test", + Columns: []Column{ + { + Name: "_cq_id", + Type: types.ExtensionTypes.UUID, + Description: "Internal CQ ID of the row", + NotNull: true, + Unique: true, + }, + {Name: "bool", Type: arrow.FixedWidthTypes.Boolean, PrimaryKey: true}, + }, + }, + expectedChanges: []TableColumnChange{ + { + Type: TableColumnChangeTypeMoveToCQOnly, + }, + { + Type: TableColumnChangeTypeUpdate, + ColumnName: "_cq_id", + Current: Column{ + Name: "_cq_id", + Type: types.ExtensionTypes.UUID, + Description: "Internal CQ ID of the row", + NotNull: true, + Unique: true, + PrimaryKey: true, + }, + Previous: Column{ + Name: "_cq_id", + Type: types.ExtensionTypes.UUID, + Description: "Internal CQ ID of the row", + NotNull: true, + Unique: true, + }, + }, + { + Type: TableColumnChangeTypeUpdate, + ColumnName: "bool", + Current: Column{Name: "bool", Type: arrow.FixedWidthTypes.Boolean, PrimaryKey: false}, + Previous: Column{Name: "bool", Type: arrow.FixedWidthTypes.Boolean, PrimaryKey: true}, + }, + }, + }, + + { + name: "move to cq_id as primary key and drop unique constraint", + target: &Table{ + Name: "test", + Columns: []Column{ + { + Name: "_cq_id", + Type: types.ExtensionTypes.UUID, + Description: "Internal CQ ID of the row", + NotNull: true, + // Unique: true, + PrimaryKey: true, + }, + {Name: "bool", Type: arrow.FixedWidthTypes.Boolean, PrimaryKey: false}, + }, + }, + source: &Table{ + Name: "test", + Columns: []Column{ + { + Name: "_cq_id", + Type: types.ExtensionTypes.UUID, + Description: "Internal CQ ID of the row", + NotNull: true, + Unique: true, + }, + {Name: "bool", Type: arrow.FixedWidthTypes.Boolean, PrimaryKey: true}, + }, + }, + expectedChanges: []TableColumnChange{ + { + Type: TableColumnChangeTypeMoveToCQOnly, + }, + { + Type: TableColumnChangeTypeUpdate, + ColumnName: "_cq_id", + Current: Column{ + Name: "_cq_id", + Type: types.ExtensionTypes.UUID, + Description: "Internal CQ ID of the row", + NotNull: true, + Unique: false, + PrimaryKey: true, + }, + Previous: Column{ + Name: "_cq_id", + Type: types.ExtensionTypes.UUID, + Description: "Internal CQ ID of the row", + NotNull: true, + Unique: true, + }, + }, + { + Type: TableColumnChangeTypeRemoveUniqueConstraint, + ColumnName: "_cq_id", + Previous: Column{ + Name: "_cq_id", + Type: types.ExtensionTypes.UUID, + Description: "Internal CQ ID of the row", + NotNull: true, + Unique: true, + }, + }, + { + Type: TableColumnChangeTypeUpdate, + ColumnName: "bool", + Current: Column{Name: "bool", Type: arrow.FixedWidthTypes.Boolean, PrimaryKey: false}, + Previous: Column{Name: "bool", Type: arrow.FixedWidthTypes.Boolean, PrimaryKey: true}, + }, + }, + }, + + { + name: "drop unique constraint", + target: &Table{ + Name: "test", + Columns: []Column{ + { + Name: "_cq_id", + Type: types.ExtensionTypes.UUID, + Description: "Internal CQ ID of the row", + NotNull: true, + }, + {Name: "bool", Type: arrow.FixedWidthTypes.Boolean, PrimaryKey: true}, + }, + }, + source: &Table{ + Name: "test", + Columns: []Column{ + { + Name: "_cq_id", + Type: types.ExtensionTypes.UUID, + Description: "Internal CQ ID of the row", + NotNull: true, + Unique: true, + }, + {Name: "bool", Type: arrow.FixedWidthTypes.Boolean, PrimaryKey: true}, + }, + }, + expectedChanges: []TableColumnChange{ + { + Type: TableColumnChangeTypeRemoveUniqueConstraint, + ColumnName: "_cq_id", + Previous: Column{ + Name: "_cq_id", + Type: types.ExtensionTypes.UUID, + Description: "Internal CQ ID of the row", + NotNull: true, + Unique: true, + }, + }, + }, + }, +} + +func TestTableGetChanges(t *testing.T) { + for _, tc := range testTableGetChangeTestCases { + t.Run(tc.name, func(t *testing.T) { + changes := tc.target.GetChanges(tc.source) + if diff := cmp.Diff(changes, tc.expectedChanges); diff != "" { + t.Errorf("diff (+got, -want): %v", diff) + } + }) + } +} + +func TestTableToAndFromArrow(t *testing.T) { + // The attributes in this table should all be preserved when converting to and from Arrow. + table := &Table{ + Name: "test_table", + Description: "Test table description", + Title: "Test Table", + Parent: &Table{ + Name: "parent_table", + }, + IsIncremental: true, + Columns: []Column{ + {Name: "bool", Type: arrow.FixedWidthTypes.Boolean}, + {Name: "int", Type: arrow.PrimitiveTypes.Int64}, + {Name: "float", Type: arrow.PrimitiveTypes.Float64}, + {Name: "string", Type: arrow.BinaryTypes.String}, + {Name: "json", Type: types.ExtensionTypes.JSON}, + {Name: "unique", Type: arrow.BinaryTypes.String, Unique: true}, + {Name: "primary_key", Type: arrow.BinaryTypes.String, PrimaryKey: true}, + {Name: "not_null", Type: arrow.BinaryTypes.String, NotNull: true}, + {Name: "incremental_key", Type: arrow.BinaryTypes.String, IncrementalKey: true}, + {Name: "multiple_attributes", Type: arrow.BinaryTypes.String, PrimaryKey: true, IncrementalKey: true, NotNull: true, Unique: true}, + }, + } + arrowSchema := table.ToArrowSchema() + tableFromArrow, err := NewTableFromArrowSchema(arrowSchema) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(table, tableFromArrow); diff != "" { + t.Errorf("diff (+got, -want): %v", diff) + } +} + +func TestValidateDuplicateTables(t *testing.T) { + tests := []struct { + name string + tables Tables + err string + }{ + { + name: "should return error when duplicate tables are found", + tables: Tables{{Name: "table1"}, {Name: "table1"}}, + err: "duplicate table table1", + }, + { + name: "should return error when duplicate relational tables are found", + tables: Tables{{Name: "table1", Relations: []*Table{{Name: "table2"}, {Name: "table2"}}}}, + err: "duplicate table table2", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := tc.tables.ValidateDuplicateTables() + if tc.err != "" { + require.ErrorContains(t, err, tc.err) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/plugin-sdk/schema/testdata.go b/plugin-sdk/schema/testdata.go new file mode 100644 index 00000000..0c6775b3 --- /dev/null +++ b/plugin-sdk/schema/testdata.go @@ -0,0 +1,497 @@ +package schema + +import ( + "crypto/sha256" + "encoding/base64" + "fmt" + "strconv" + "strings" + "time" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/google/uuid" + "golang.org/x/exp/rand" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" +) + +// TestSourceOptions controls which types are included by TestSourceColumns. +type TestSourceOptions struct { + SkipDates bool + SkipDurations bool + SkipIntervals bool + SkipLargeTypes bool // e.g. large binary, large string + SkipLists bool // lists of all primitive types. Lists that were supported by CQTypes are always included. + SkipMaps bool + SkipStructs bool + SkipTimes bool // time of day types + SkipTimestamps bool // timestamp types. Microsecond timestamp is always be included, regardless of this setting. + TimePrecision time.Duration + SkipDecimals bool +} + +// listOfColumns returns a list of columns that are lists of the given columns. +func listOfColumns(baseColumns []Column) []Column { + columns := make([]Column, len(baseColumns)) + for i := 0; i < len(baseColumns); i++ { + columns[i] = Column{Name: baseColumns[i].Name + "_list", Type: arrow.ListOf(baseColumns[i].Type)} + } + return columns +} + +// mapOfColumns returns a list of columns that are maps of the given columns. +// nolint:unused +func mapOfColumns(baseColumns []Column) []Column { + columns := make([]Column, len(baseColumns)*2) + for i := 0; i < len(columns); i += 2 { + // we focus on string and int keys for now + n := i / 2 + columns[i] = Column{Name: "string_" + baseColumns[n].Name + "_map", Type: arrow.MapOf(arrow.BinaryTypes.String, baseColumns[n].Type)} + columns[i+1] = Column{Name: "int_" + baseColumns[n].Name + "_map", Type: arrow.MapOf(arrow.PrimitiveTypes.Int64, baseColumns[n].Type)} + } + return columns +} + +func columnsToFields(columns []Column) []arrow.Field { + fields := make([]arrow.Field, len(columns)) + for i := range columns { + fields[i] = arrow.Field{ + Name: columns[i].Name, + Type: columns[i].Type, + } + } + return fields +} + +func TestTable(name string, opts TestSourceOptions) *Table { + t := &Table{ + Name: name, + Columns: make(ColumnList, 0), + } + var columns ColumnList + columns = append(columns, ColumnList{ + // id is to be used as an auto-incrementing column that can be used to order the rows in tests + {Name: "id", Type: arrow.PrimitiveTypes.Int64, NotNull: true}, + + // primitive columns + {Name: "int8", Type: arrow.PrimitiveTypes.Int8}, + {Name: "int16", Type: arrow.PrimitiveTypes.Int16}, + {Name: "int32", Type: arrow.PrimitiveTypes.Int32}, + {Name: "int64", Type: arrow.PrimitiveTypes.Int64}, + {Name: "uint8", Type: arrow.PrimitiveTypes.Uint8}, + {Name: "uint16", Type: arrow.PrimitiveTypes.Uint16}, + {Name: "uint32", Type: arrow.PrimitiveTypes.Uint32}, + {Name: "uint64", Type: arrow.PrimitiveTypes.Uint64}, + {Name: "float32", Type: arrow.PrimitiveTypes.Float32}, + {Name: "float64", Type: arrow.PrimitiveTypes.Float64}, + + // basic columns + {Name: "binary", Type: arrow.BinaryTypes.Binary}, + {Name: "string", Type: arrow.BinaryTypes.String}, + {Name: "boolean", Type: arrow.FixedWidthTypes.Boolean}, + + // extension types + {Name: "uuid", Type: types.ExtensionTypes.UUID}, + {Name: "inet", Type: types.ExtensionTypes.Inet}, + {Name: "mac", Type: types.ExtensionTypes.MAC}, + {Name: "json", Type: types.ExtensionTypes.JSON}, + }...) + if !opts.SkipDates { + columns = append(columns, ColumnList{ + {Name: "date32", Type: arrow.FixedWidthTypes.Date32}, + {Name: "date64", Type: arrow.FixedWidthTypes.Date64}, + }...) + } + if !opts.SkipDurations { + columns = append(columns, ColumnList{ + {Name: "duration_s", Type: arrow.FixedWidthTypes.Duration_s}, + {Name: "duration_ms", Type: arrow.FixedWidthTypes.Duration_ms}, + {Name: "duration_us", Type: arrow.FixedWidthTypes.Duration_us}, + {Name: "duration_ns", Type: arrow.FixedWidthTypes.Duration_ns}, + }...) + } + + if !opts.SkipIntervals { + columns = append(columns, ColumnList{ + {Name: "interval_month", Type: arrow.FixedWidthTypes.MonthInterval}, + {Name: "interval_day_time", Type: arrow.FixedWidthTypes.DayTimeInterval}, + {Name: "interval_month_day_nano", Type: arrow.FixedWidthTypes.MonthDayNanoInterval}, + }...) + } + + if !opts.SkipLargeTypes { + columns = append(columns, ColumnList{ + {Name: "large_binary", Type: arrow.BinaryTypes.LargeBinary}, + {Name: "large_string", Type: arrow.BinaryTypes.LargeString}, + }...) + } + + if !opts.SkipTimes { + columns = append(columns, ColumnList{ + {Name: "time32_s", Type: arrow.FixedWidthTypes.Time32s}, + {Name: "time32_ms", Type: arrow.FixedWidthTypes.Time32ms}, + {Name: "time64_us", Type: arrow.FixedWidthTypes.Time64us}, + {Name: "time64_ns", Type: arrow.FixedWidthTypes.Time64ns}, + }...) + } + + if !opts.SkipTimestamps { + columns = append(columns, ColumnList{ + {Name: "timestamp_s", Type: arrow.FixedWidthTypes.Timestamp_s}, + {Name: "timestamp_ms", Type: arrow.FixedWidthTypes.Timestamp_ms}, + {Name: "timestamp_us", Type: arrow.FixedWidthTypes.Timestamp_us}, + {Name: "timestamp_ns", Type: arrow.FixedWidthTypes.Timestamp_ns}, + }...) + } + + if !opts.SkipDecimals { + columns = append(columns, ColumnList{ + {Name: "decimal128", Type: &arrow.Decimal128Type{Precision: 19, Scale: 10}}, + // {Name: "decimal256", Type: &arrow.Decimal256Type{Precision: 40, Scale: 10}}, + }...) + } + + if !opts.SkipStructs { + columns = append(columns, Column{Name: "struct", Type: arrow.StructOf(columnsToFields(columns)...)}) + + // struct with nested struct + // columns = append(columns, Column{Name: "nested_struct", Type: arrow.StructOf(arrow.Field{Name: "inner", Type: arrow.StructOf(columnsToFields(basicColumns...)...)})}) + } + + if !opts.SkipLists { + cols := excludeType(columns, types.ExtensionTypes.JSON) + columns = append(columns, listOfColumns(cols)...) + } + + if !opts.SkipMaps { + columns = append(columns, mapOfColumns(columns)...) + } + + t.Columns = append(t.Columns, columns...) + return t +} + +func excludeType(columns ColumnList, typ arrow.DataType) ColumnList { + var cols ColumnList + for _, c := range columns { + if !arrow.TypeEqual(c.Type, typ) { + cols = append(cols, c) + } + } + return cols +} + +// GenTestDataOptions are options for generating test data +type GenTestDataOptions struct { + // SourceName is the name of the source to set in the source_name column. + SourceName string + // SyncTime is the time to set in the sync_time column. + SyncTime time.Time + // MaxRows is the number of rows to generate. + MaxRows int + // StableTime is the time to use for all rows other than sync time. If set to time.Time{}, a new time will be generated + StableTime time.Time + // TimePrecision is the precision to use for time columns. + TimePrecision time.Duration + // NullRows indicates whether to generate rows with all null values. + NullRows bool + // UseHomogeneousType indicates whether to use a single type for JSON arrays. + UseHomogeneousType bool +} + +type TestDataGenerator struct { + counter int + seed uint64 + colToRnd map[string]*rand.Rand +} + +func NewTestDataGenerator(randomSeed uint64) *TestDataGenerator { + return &TestDataGenerator{ + counter: int(randomSeed), + seed: randomSeed, + colToRnd: map[string]*rand.Rand{}, + } +} + +func (tg *TestDataGenerator) Reset() { + tg.counter = 0 + tg.colToRnd = map[string]*rand.Rand{} +} + +// Generate will produce a single arrow.Record with the given schema and options. +func (tg *TestDataGenerator) Generate(table *Table, opts GenTestDataOptions) arrow.Record { + sc := table.ToArrowSchema() + if opts.MaxRows == 0 { + // We generate an empty record + bldr := array.NewRecordBuilder(memory.DefaultAllocator, sc) + defer bldr.Release() + return bldr.NewRecord() + } + + var records []arrow.Record + for j := 0; j < opts.MaxRows; j++ { + tg.counter++ + bldr := array.NewRecordBuilder(memory.DefaultAllocator, sc) + for i, c := range table.Columns { + if opts.NullRows && !c.NotNull && !c.PrimaryKey && + c.Name != CqSourceNameColumn.Name && + c.Name != CqSyncTimeColumn.Name && + c.Name != CqIDColumn.Name { + bldr.Field(i).AppendNull() + continue + } + example := tg.getExampleJSON(c.Name, c.Type, opts) + l := `[` + example + `]` + err := bldr.Field(i).UnmarshalJSON([]byte(l)) + if err != nil { + panic(fmt.Sprintf("failed to unmarshal json `%v` for column %v: %v", l, c.Name, err)) + } + } + records = append(records, bldr.NewRecord()) + bldr.Release() + } + + arrowTable := array.NewTableFromRecords(sc, records) + columns := make([]arrow.Array, sc.NumFields()) + for n := 0; n < sc.NumFields(); n++ { + concatenated, err := array.Concatenate(arrowTable.Column(n).Data().Chunks(), memory.DefaultAllocator) + if err != nil { + panic(fmt.Sprintf("failed to concatenate arrays: %v", err)) + } + columns[n] = concatenated + } + + return array.NewRecord(sc, columns, -1) +} + +func (tg TestDataGenerator) getExampleJSON(colName string, dataType arrow.DataType, opts GenTestDataOptions) string { + var rnd, found = tg.colToRnd[colName] + if !found { + tg.colToRnd[colName] = rand.New(rand.NewSource(tg.seed)) + rnd = tg.colToRnd[colName] + } + + // special case for auto-incrementing id column, used for to determine ordering in tests + if arrow.IsInteger(dataType.ID()) && colName == "id" { + return `` + strconv.Itoa(tg.counter) + `` + } + + // handle lists (including maps) + if arrow.IsListLike(dataType.ID()) { + if dataType.ID() == arrow.MAP { + k := tg.getExampleJSON(colName, dataType.(*arrow.MapType).KeyType(), opts) + v := tg.getExampleJSON(colName, dataType.(*arrow.MapType).ItemType(), opts) + k2 := tg.getExampleJSON(colName, dataType.(*arrow.MapType).KeyType(), opts) + v2 := tg.getExampleJSON(colName, dataType.(*arrow.MapType).ItemType(), opts) + return fmt.Sprintf(`[{"key": %s,"value": %s},{"key": %s,"value": %s}]`, k, v, k2, v2) + } + inner := dataType.(*arrow.ListType).Elem() + return `[` + tg.getExampleJSON(colName, inner, opts) + `,null,` + tg.getExampleJSON(colName, inner, opts) + `]` + } + // handle extension types + if arrow.TypeEqual(dataType, types.ExtensionTypes.UUID) { + // This will make UUIDs deterministic like all other types + hash := sha256.New() + hash.Write([]byte(fmt.Sprintf(`"AString%d"`, rnd.Intn(100000)))) + u := uuid.NewSHA1(uuid.UUID{}, hash.Sum(nil)) + return `"` + u.String() + `"` + } + if arrow.TypeEqual(dataType, types.ExtensionTypes.JSON) { + if strings.HasSuffix(colName, "_array") { + if opts.UseHomogeneousType { + return `[{"test1":"test1"},{"test2":"test2"},{"test3":"test3"}]` + } + return `[{"test":"test"},123,{"test_number":456}]` + } + if opts.UseHomogeneousType { + return `{"test":["a", "b", "c"]}` + } + return `{"test":["a","b",3]}` + } + if arrow.TypeEqual(dataType, types.ExtensionTypes.Inet) { + return `"192.0.2.0/24"` + } + if arrow.TypeEqual(dataType, types.ExtensionTypes.MAC) { + return `"aa:bb:cc:dd:ee:ff"` + } + + // handle signed integers + if arrow.IsSignedInteger(dataType.ID()) { + switch dataType { + case arrow.PrimitiveTypes.Int8: + return fmt.Sprintf("-%d", rnd.Intn(int(^uint8(0)>>1))) + case arrow.PrimitiveTypes.Int16: + return fmt.Sprintf("-%d", rnd.Intn(int(^uint16(0)>>1))) + case arrow.PrimitiveTypes.Int32: + return fmt.Sprintf("-%d", rnd.Intn(int(^uint32(0)>>1))) + case arrow.PrimitiveTypes.Int64: + return fmt.Sprintf("-%d", rnd.Int63n(int64(^uint64(0)>>1))) + } + } + + // handle unsigned integers + if arrow.IsUnsignedInteger(dataType.ID()) { + switch dataType { + case arrow.PrimitiveTypes.Uint8: + return fmt.Sprintf("%d", rnd.Uint64n(uint64(^uint8(0)))) + case arrow.PrimitiveTypes.Uint16: + return fmt.Sprintf("%d", rnd.Uint64n(uint64(^uint16(0)))) + case arrow.PrimitiveTypes.Uint32: + return fmt.Sprintf("%d", rnd.Uint64n(uint64(^uint32(0)))) + case arrow.PrimitiveTypes.Uint64: + return fmt.Sprintf("%d", rnd.Uint64()) + } + } + + // handle floats + if arrow.IsFloating(dataType.ID()) { + return fmt.Sprintf("%d.%d", rnd.Intn(1e3), rnd.Intn(1e3)) + } + + // handle decimals + if arrow.IsDecimal(dataType.ID()) { + return fmt.Sprintf("%d.%d", rnd.Int63n(1e9), rnd.Int63n(1e10)) + } + + // handle booleans + if arrow.TypeEqual(dataType, arrow.FixedWidthTypes.Boolean) { + if rnd.Intn(2) == 0 { + return "false" + } + return "true" + } + + // handle strings + stringTypes := []arrow.DataType{ + arrow.BinaryTypes.String, + arrow.BinaryTypes.LargeString, + } + for _, stringType := range stringTypes { + if arrow.TypeEqual(dataType, stringType) { + if colName == CqSourceNameColumn.Name { + return `"` + opts.SourceName + `"` + } + n := rnd.Intn(100000) + return fmt.Sprintf(`"AString%d"`, n) + } + } + + // handle binary types + binaryTypes := []arrow.DataType{ + arrow.BinaryTypes.Binary, + arrow.BinaryTypes.LargeBinary, + } + for _, binaryType := range binaryTypes { + if arrow.TypeEqual(dataType, binaryType) { + bytes := make([]byte, 4) + rnd.Read(bytes) + return `"` + base64.StdEncoding.EncodeToString(bytes) + `"` + } + } + + // handle structs + if dataType.ID() == arrow.STRUCT { + var columns []string + for _, field := range dataType.(*arrow.StructType).Fields() { + v := tg.getExampleJSON(field.Name, field.Type, opts) + columns = append(columns, fmt.Sprintf(`"%s": %v`, field.Name, v)) + } + return `{` + strings.Join(columns, ",") + `}` + } + + // handle timestamp types + timestampTypes := []arrow.DataType{ + arrow.FixedWidthTypes.Timestamp_s, + arrow.FixedWidthTypes.Timestamp_ms, + arrow.FixedWidthTypes.Timestamp_us, + arrow.FixedWidthTypes.Timestamp_ns, + arrow.FixedWidthTypes.Time32s, + arrow.FixedWidthTypes.Time32ms, + arrow.FixedWidthTypes.Time64us, + arrow.FixedWidthTypes.Time64ns, + } + for _, timestampType := range timestampTypes { + if arrow.TypeEqual(dataType, timestampType) { + t := time.Now() + if colName == CqSyncTimeColumn.Name { + t = opts.SyncTime.UTC() + } else if !opts.StableTime.IsZero() { + t = opts.StableTime + } + t = t.Truncate(opts.TimePrecision) + + switch timestampType { + case arrow.FixedWidthTypes.Timestamp_s: + return strconv.FormatInt(t.Unix(), 10) + case arrow.FixedWidthTypes.Timestamp_ms: + return strconv.FormatInt(t.UnixMilli(), 10) + case arrow.FixedWidthTypes.Timestamp_us: + return strconv.FormatInt(t.UnixMicro(), 10) + case arrow.FixedWidthTypes.Timestamp_ns: + // Note: We use microseconds instead of nanoseconds here because + // nanosecond precision is not supported by many destinations. + // For now, we begrudgingly accept loss of precision in these cases. + // See https://github.com/cloudquery/plugin-sdk/issues/830 + t = t.Truncate(time.Microsecond) + // Use string timestamp string format here because JSON integers are + // unmarshalled as float64, losing precision for nanosecond timestamps. + return t.Format(`"2006-01-02 15:04:05.999999999"`) + case arrow.FixedWidthTypes.Time32s: + h, m, s := t.Clock() + return strconv.Itoa(h*3600 + m*60 + s) + case arrow.FixedWidthTypes.Time32ms: + h, m, s := t.Clock() + ns := t.Nanosecond() + return strconv.Itoa(h*3600000 + m*60000 + s*1000 + ns/1000000) + case arrow.FixedWidthTypes.Time64us: + h, m, s := t.Clock() + ns := t.Nanosecond() + return strconv.Itoa(h*3600000000 + m*60000000 + s*1000000 + ns/1000) + case arrow.FixedWidthTypes.Time64ns: + h, m, s := t.Clock() + ns := t.Nanosecond() + return strconv.Itoa(h*3600000000000 + m*60000000000 + s*1000000000 + ns) + default: + panic("unhandled timestamp type: " + timestampType.Name()) + } + } + } + + // handle date types + if arrow.TypeEqual(dataType, arrow.FixedWidthTypes.Date32) { + return fmt.Sprintf("%d", 19471+rnd.Intn(100)) + } + if arrow.TypeEqual(dataType, arrow.FixedWidthTypes.Date64) { + ms := (19471 + rnd.Intn(100)) * 86400000 + return fmt.Sprintf("%d", ms) + } + + // handle duration and interval types + if arrow.TypeEqual(dataType, arrow.FixedWidthTypes.DayTimeInterval) { + n := rnd.Intn(10000) + return fmt.Sprintf(`{"days": %[1]d, "milliseconds": %[1]d}`, n) + } + if arrow.TypeEqual(dataType, arrow.FixedWidthTypes.MonthInterval) { + return `{"months": 1}` + } + if arrow.TypeEqual(dataType, arrow.FixedWidthTypes.MonthDayNanoInterval) { + n := rnd.Intn(10000) + return fmt.Sprintf(`{"months": %[1]d, "days": %[1]d, "nanoseconds": %[1]d}`, n) + } + durationTypes := []arrow.DataType{ + arrow.FixedWidthTypes.Duration_s, + arrow.FixedWidthTypes.Duration_ms, + arrow.FixedWidthTypes.Duration_us, + arrow.FixedWidthTypes.Duration_ns, + } + for _, durationType := range durationTypes { + if arrow.TypeEqual(dataType, durationType) { + n := rnd.Intn(10000000) + return fmt.Sprintf("%d", n) + } + } + + panic("unknown type: " + dataType.String() + " column: " + colName) +} diff --git a/plugin-sdk/schema/testdata_test.go b/plugin-sdk/schema/testdata_test.go new file mode 100644 index 00000000..b246974e --- /dev/null +++ b/plugin-sdk/schema/testdata_test.go @@ -0,0 +1,54 @@ +package schema + +import "testing" + +func TestTestSourceColumns_Default(t *testing.T) { + // basic sanity check for tested columns + table := TestTable("test", TestSourceOptions{}) + if len(table.Columns) < 73 { + t.Fatalf("expected at least 73 columns by default got: %d ", len(table.Columns)) + } + // test some specific columns + checkColumnsExist(t, table.Columns, []string{"int64", "date32", "timestamp_us", "string", "struct", "string_list"}) +} + +func TestTestSourceColumns_SkipAll(t *testing.T) { + table := TestTable("test", TestSourceOptions{ + SkipLists: true, + SkipTimestamps: true, + SkipDates: true, + SkipMaps: true, + SkipStructs: true, + SkipIntervals: true, + SkipDurations: true, + SkipTimes: true, + SkipLargeTypes: true, + }) + + // test some specific columns + checkColumnsExist(t, table.Columns, []string{"int64", "string"}) + checkColumnsDontExist(t, table.Columns, []string{"date32", "struct", "string_map"}) +} + +func checkColumnsExist(t *testing.T, list ColumnList, cols []string) { + for _, col := range cols { + if list.Get(col) == nil { + t.Errorf("expected column %s to be present", col) + } + } +} + +func checkColumnsDontExist(t *testing.T, list ColumnList, cols []string) { + for _, col := range cols { + if list.Get(col) != nil { + t.Errorf("expected no %s column", col) + } + } +} + +func TestGenTestData(*testing.T) { + table := TestTable("test", TestSourceOptions{}) + // smoke test that no panics + tg := NewTestDataGenerator(0) + _ = tg.Generate(table, GenTestDataOptions{}) +} diff --git a/plugin-sdk/schema/validators.go b/plugin-sdk/schema/validators.go new file mode 100644 index 00000000..f6764697 --- /dev/null +++ b/plugin-sdk/schema/validators.go @@ -0,0 +1,54 @@ +package schema + +import ( + "encoding/json" + + "github.com/apache/arrow/go/v16/arrow" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" +) + +func FindEmptyColumns(table *Table, records []arrow.Record) []string { + columnsWithValues := make([]bool, len(table.Columns)) + emptyColumns := make([]string, 0) + + for _, resource := range records { + for colIndex, arr := range resource.Columns() { + for i := 0; i < arr.Len(); i++ { + if arr.IsValid(i) { + if arrow.TypeEqual(arr.DataType(), types.ExtensionTypes.JSON) { + // JSON column shouldn't be empty + val := arr.GetOneForMarshal(i).(json.RawMessage) + if isEmptyJSON(val) { + continue + } + } + + columnsWithValues[colIndex] = true + } + } + } + } + + // Make sure every column has at least one value. + for i, hasValue := range columnsWithValues { + col := table.Columns[i] + emptyExpected := col.Name == "_cq_parent_id" && table.Parent == nil + if !hasValue && !emptyExpected && !col.IgnoreInTests { + emptyColumns = append(emptyColumns, col.Name) + } + } + return emptyColumns +} + +func isEmptyJSON(msg json.RawMessage) bool { + if len(msg) == 0 { + return true + } + switch string(msg) { + case "null", "{}", "[]": + return true + default: + return false + } +} diff --git a/plugin-sdk/schema/validators_test.go b/plugin-sdk/schema/validators_test.go new file mode 100644 index 00000000..e8facc34 --- /dev/null +++ b/plugin-sdk/schema/validators_test.go @@ -0,0 +1,57 @@ +package schema + +import ( + "fmt" + "testing" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" +) + +func TestFindEmptyColumns(t *testing.T) { + table := TestTable("test", TestSourceOptions{}) + tg := NewTestDataGenerator(0) + record := tg.Generate(table, GenTestDataOptions{ + MaxRows: 1, + NullRows: true, + }) + v := FindEmptyColumns(table, []arrow.Record{record}) + require.NotEmpty(t, v) + require.Len(t, v, len(table.Columns)-1) // exclude "id" +} + +func TestFindEmptyColumnsNotEmpty(t *testing.T) { + table := TestTable("test", TestSourceOptions{}) + tg := NewTestDataGenerator(0) + record := tg.Generate(table, GenTestDataOptions{ + MaxRows: 1, + NullRows: false, + }) + v := FindEmptyColumns(table, []arrow.Record{record}) + require.Empty(t, v) +} + +func TestFindEmptyColumnsJSON(t *testing.T) { + table := &Table{ + Name: "test", + Columns: ColumnList{ + {Name: "json", Type: types.ExtensionTypes.JSON}, + }, + } + sc := table.ToArrowSchema() + bldr := array.NewRecordBuilder(memory.DefaultAllocator, sc) + err := bldr.Field(0).UnmarshalJSON([]byte(`[{}]`)) + if err != nil { + panic(fmt.Sprintf("failed to unmarshal json for column: %v", err)) + } + records := []arrow.Record{bldr.NewRecord()} + bldr.Release() + + v := FindEmptyColumns(table, records) + require.NotEmpty(t, v) + require.Len(t, v, 1) +} diff --git a/plugin-sdk/serve/constants.go b/plugin-sdk/serve/constants.go new file mode 100644 index 00000000..95971f7d --- /dev/null +++ b/plugin-sdk/serve/constants.go @@ -0,0 +1,12 @@ +package serve + +import ( + "time" +) + +const ( + // bufSize used for unit testing grpc server and client + testBufSize = 1024 * 1024 + flushTimeout = 5 * time.Second + MaxMsgSize = 100 * 1024 * 1024 // 100 MiB +) diff --git a/plugin-sdk/serve/destination_v1_test.go b/plugin-sdk/serve/destination_v1_test.go new file mode 100644 index 00000000..5ab55157 --- /dev/null +++ b/plugin-sdk/serve/destination_v1_test.go @@ -0,0 +1,183 @@ +package serve + +import ( + "bytes" + "context" + "encoding/json" + "sync" + "testing" + "time" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/ipc" + "github.com/apache/arrow/go/v16/arrow/memory" + pb "github.com/cloudquery/plugin-pb-go/pb/destination/v1" + pbSource "github.com/cloudquery/plugin-pb-go/pb/source/v2" + "github.com/cloudquery/plugin-pb-go/specs" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/internal/memdb" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/plugin" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +func TestDestinationV1(t *testing.T) { + p := plugin.NewPlugin("testDestinationPlugin", "development", memdb.NewMemDBClient) + srv := Plugin(p, WithArgs("serve"), WithDestinationV0V1Server(), WithTestListener()) + ctx := context.Background() + ctx, cancel := context.WithCancel(ctx) + var wg sync.WaitGroup + wg.Add(1) + var serverErr error + go func() { + defer wg.Done() + serverErr = srv.Serve(ctx) + }() + defer func() { + cancel() + wg.Wait() + }() + + // https://stackoverflow.com/questions/42102496/testing-a-grpc-service + // TODO: Remove once there's a documented migration path per https://github.com/grpc/grpc-go/issues/7244 + // nolint:staticcheck + conn, err := grpc.DialContext(ctx, "bufnet", grpc.WithContextDialer(srv.bufPluginDialer), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithBlock()) + if err != nil { + t.Fatalf("Failed to dial bufnet: %v", err) + } + c := pb.NewDestinationClient(conn) + spec := specs.Destination{ + WriteMode: specs.WriteModeAppend, + } + specBytes, err := json.Marshal(spec) + if err != nil { + t.Fatal(err) + } + if _, err := c.Configure(ctx, &pb.Configure_Request{Config: specBytes}); err != nil { + t.Fatal(err) + } + + getNameRes, err := c.GetName(ctx, &pb.GetName_Request{}) + if err != nil { + t.Fatal(err) + } + if getNameRes.Name != "testDestinationPlugin" { + t.Fatalf("expected name to be testDestinationPlugin but got %s", getNameRes.Name) + } + + getVersionRes, err := c.GetVersion(ctx, &pb.GetVersion_Request{}) + if err != nil { + t.Fatal(err) + } + if getVersionRes.Version != "development" { + t.Fatalf("expected version to be development but got %s", getVersionRes.Version) + } + + tableName := "test_destination_serve" + sourceName := "test_destination_serve_source" + syncTime := time.Now() + table := &schema.Table{ + Name: tableName, + Columns: []schema.Column{ + schema.CqSourceNameColumn, + schema.CqSyncTimeColumn, + {Name: "col1", Type: arrow.PrimitiveTypes.Int16}, + }, + } + + tables := schema.Tables{table} + sourceSpec := specs.Source{ + Name: sourceName, + } + schemas := tables.ToArrowSchemas() + encodedTables, err := pbSource.SchemasToBytes(schemas) + if err != nil { + t.Fatal(err) + } + + if _, err := c.Migrate(ctx, &pb.Migrate_Request{ + Tables: encodedTables, + }); err != nil { + t.Fatal(err) + } + bldr := array.NewRecordBuilder(memory.DefaultAllocator, table.ToArrowSchema()) + bldr.Field(0).(*array.StringBuilder).Append(sourceName) + bldr.Field(1).(*array.TimestampBuilder).AppendTime(syncTime) + bldr.Field(2).(*array.Int16Builder).Append(1) + rec := bldr.NewRecord() + + sourceSpecBytes, err := json.Marshal(sourceSpec) + if err != nil { + t.Fatal(err) + } + writeClient, err := c.Write(ctx) + if err != nil { + t.Fatal(err) + } + if err := writeClient.Send(&pb.Write_Request{ + SourceSpec: sourceSpecBytes, + Source: sourceSpec.Name, + Timestamp: timestamppb.New(syncTime.Truncate(time.Microsecond)), + Tables: encodedTables, + }); err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + wr := ipc.NewWriter(&buf, ipc.WithSchema(rec.Schema())) + if err := wr.Write(rec); err != nil { + t.Fatal(err) + } + if err := wr.Close(); err != nil { + t.Fatal(err) + } + if err := writeClient.Send(&pb.Write_Request{ + Resource: buf.Bytes(), + }); err != nil { + t.Fatal(err) + } + + if _, err := writeClient.CloseAndRecv(); err != nil { + t.Fatal(err) + } + // serversDestination + msgs, err := p.SyncAll(ctx, plugin.SyncOptions{ + Tables: []string{tableName}, + }) + if err != nil { + t.Fatal(err) + } + totalResources := 0 + for _, msg := range msgs { + totalResources++ + m := msg.(*message.SyncInsert) + if !array.RecordEqual(rec, m.Record) { + // diff := plugin.RecordDiff(rec, resource) + // t.Fatalf("diff at %d: %s", totalResources, diff) + t.Fatalf("expected %v but got %v", rec, m.Record) + } + } + if totalResources != 1 { + t.Fatalf("expected 1 resource but got %d", totalResources) + } + if _, err := c.DeleteStale(ctx, &pb.DeleteStale_Request{ + Source: "testSource", + Timestamp: timestamppb.New(time.Now().Truncate(time.Microsecond)), + Tables: encodedTables, + }); err != nil { + t.Fatal(err) + } + + if _, err := c.Close(ctx, &pb.Close_Request{}); err != nil { + t.Fatalf("failed to call Close: %v", err) + } + + cancel() + wg.Wait() + if serverErr != nil { + t.Fatal(serverErr) + } +} diff --git a/plugin-sdk/serve/docs.go b/plugin-sdk/serve/docs.go new file mode 100644 index 00000000..593a7b45 --- /dev/null +++ b/plugin-sdk/serve/docs.go @@ -0,0 +1,56 @@ +package serve + +import ( + "fmt" + "strings" + + "github.com/spf13/cobra" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/docs" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/plugin" +) + +const ( + pluginDocShort = "Generate documentation for tables" + pluginDocLong = `Generate documentation for tables + +If format is markdown, a destination directory will be created (if necessary) containing markdown files. +Example: +doc ./output + +If format is JSON, a destination directory will be created (if necessary) with a single json file called __tables.json. +Example: +doc --format json . +` +) + +func (s *PluginServe) newCmdPluginDoc() *cobra.Command { + format := newEnum([]string{"json", "markdown"}, "markdown") + cmd := &cobra.Command{ + Use: "doc ", + Short: pluginDocShort, + Long: pluginDocLong, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if err := s.plugin.Init(cmd.Context(), nil, plugin.NewClientOptions{ + NoConnection: true, + }); err != nil { + return err + } + tables, err := s.plugin.Tables(cmd.Context(), plugin.TableOptions{ + Tables: []string{"*"}, + }) + if err != nil { + return err + } + g := docs.NewGenerator(s.plugin.Name(), tables) + f := docs.FormatMarkdown + if format.Value == "json" { + f = docs.FormatJSON + } + return g.Generate(args[0], f) + }, + } + cmd.Flags().Var(format, "format", fmt.Sprintf("output format. one of: %s", strings.Join(format.Allowed, ","))) + return cmd +} diff --git a/plugin-sdk/serve/docs_test.go b/plugin-sdk/serve/docs_test.go new file mode 100644 index 00000000..58f26b61 --- /dev/null +++ b/plugin-sdk/serve/docs_test.go @@ -0,0 +1,22 @@ +package serve + +import ( + "testing" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/internal/memdb" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/plugin" +) + +func TestPluginDocs(t *testing.T) { + tmpDir := t.TempDir() + p := plugin.NewPlugin( + "testPlugin", + "v1.0.0", + memdb.NewMemDBClient) + srv := Plugin(p) + cmd := srv.newCmdPluginRoot() + cmd.SetArgs([]string{"doc", tmpDir}) + if err := cmd.Execute(); err != nil { + t.Fatal(err) + } +} diff --git a/plugin-sdk/serve/enum.go b/plugin-sdk/serve/enum.go new file mode 100644 index 00000000..dc146e27 --- /dev/null +++ b/plugin-sdk/serve/enum.go @@ -0,0 +1,45 @@ +package serve + +import ( + "fmt" + "strings" +) + +// https://github.com/spf13/pflag/issues/236#issuecomment-931600452 + +type enum struct { + Allowed []string + Value string +} + +// newEnum give a list of allowed flag parameters, where the second argument is the default +func newEnum(allowed []string, d string) *enum { + return &enum{ + Allowed: allowed, + Value: d, + } +} + +func (a enum) String() string { + return a.Value +} + +func (a *enum) Set(p string) error { + isIncluded := func(opts []string, val string) bool { + for _, opt := range opts { + if val == opt { + return true + } + } + return false + } + if !isIncluded(a.Allowed, p) { + return fmt.Errorf("%s is not included in %s", p, strings.Join(a.Allowed, ",")) + } + a.Value = p + return nil +} + +func (*enum) Type() string { + return "string" +} diff --git a/plugin-sdk/serve/env.go b/plugin-sdk/serve/env.go new file mode 100644 index 00000000..966e9076 --- /dev/null +++ b/plugin-sdk/serve/env.go @@ -0,0 +1,10 @@ +package serve + +import "os" + +func getEnvOrDefault(env, def string) string { + if v := os.Getenv(env); v != "" { + return v + } + return def +} diff --git a/plugin-sdk/serve/info.go b/plugin-sdk/serve/info.go new file mode 100644 index 00000000..a087f61e --- /dev/null +++ b/plugin-sdk/serve/info.go @@ -0,0 +1,26 @@ +package serve + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +const ( + pluginInfoShort = "Print build information about this plugin" + pluginInfoLong = "Print build information about this plugin" +) + +func (s *PluginServe) newCmdPluginInfo() *cobra.Command { + cmd := &cobra.Command{ + Use: "info", + Short: pluginInfoShort, + Long: pluginInfoLong, + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, _ []string) error { + cmd.Println(fmt.Sprintf("Package and version: %s/%s/%s@%s", s.plugin.Team(), s.plugin.Kind(), s.plugin.Name(), s.plugin.Version())) + return nil + }, + } + return cmd +} diff --git a/plugin-sdk/serve/opentelemetry.go b/plugin-sdk/serve/opentelemetry.go new file mode 100644 index 00000000..f4549505 --- /dev/null +++ b/plugin-sdk/serve/opentelemetry.go @@ -0,0 +1,165 @@ +package serve + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/rs/zerolog" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.25.0" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/plugin" +) + +// newResource returns a resource describing this application. +func newResource(p *plugin.Plugin) *resource.Resource { + r, err := resource.Merge( + resource.Default(), + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceName("cloudquery-"+p.Name()), + semconv.ServiceVersion(p.Version()), + ), + ) + if err != nil { + panic(err) + } + return r +} + +func parseOtelHeaders(headers []string) map[string]string { + headerMap := make(map[string]string, len(headers)) + for _, h := range headers { + parts := strings.SplitN(h, ":", 2) + if len(parts) != 2 { + continue + } + headerMap[strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1]) + } + return headerMap +} + +type otelConfig struct { + endpoint string + insecure bool + headers []string + urlPath string +} + +func getTraceExporter(ctx context.Context, opts otelConfig) (*otlptrace.Exporter, error) { + if opts.endpoint == "" { + return nil, nil + } + + traceOptions := []otlptracehttp.Option{ + otlptracehttp.WithEndpoint(opts.endpoint), + } + + if opts.insecure { + traceOptions = append(traceOptions, otlptracehttp.WithInsecure()) + } + + if len(opts.headers) > 0 { + headers := parseOtelHeaders(opts.headers) + traceOptions = append(traceOptions, otlptracehttp.WithHeaders(headers)) + } + + if opts.urlPath != "" { + traceOptions = append(traceOptions, otlptracehttp.WithURLPath(opts.urlPath)) + } + + traceClient := otlptracehttp.NewClient(traceOptions...) + traceExporter, err := otlptrace.New(ctx, traceClient) + if err != nil { + return nil, fmt.Errorf("creating OTLP trace exporter: %w", err) + } + + return traceExporter, nil +} + +func getMetricReader(ctx context.Context, opts otelConfig) (*metric.PeriodicReader, error) { + if opts.endpoint == "" { + return nil, nil + } + + metricOptions := []otlpmetrichttp.Option{ + otlpmetrichttp.WithEndpoint(opts.endpoint), + } + + if opts.insecure { + metricOptions = append(metricOptions, otlpmetrichttp.WithInsecure()) + } + + if len(opts.headers) > 0 { + headers := parseOtelHeaders(opts.headers) + metricOptions = append(metricOptions, otlpmetrichttp.WithHeaders(headers)) + } + + if opts.urlPath != "" { + metricOptions = append(metricOptions, otlpmetrichttp.WithURLPath(opts.urlPath)) + } + + metricExporter, err := otlpmetrichttp.New(ctx, metricOptions...) + if err != nil { + return nil, fmt.Errorf("creating OTLP metric exporter: %w", err) + } + + reader := metric.NewPeriodicReader(metricExporter, metric.WithInterval(15*time.Second)) + return reader, nil +} + +func setupOtel(ctx context.Context, logger zerolog.Logger, p *plugin.Plugin, otelEndpoint string, otelEndpointInsecure bool, otelEndpointHeaders []string, otelEndpointURLPath string) (shutdown func(), err error) { + if otelEndpoint == "" { + return func() {}, nil + } + opts := otelConfig{ + endpoint: otelEndpoint, + insecure: otelEndpointInsecure, + headers: otelEndpointHeaders, + urlPath: otelEndpointURLPath, + } + traceExporter, err := getTraceExporter(ctx, opts) + if err != nil { + return nil, err + } + + metricReader, err := getMetricReader(ctx, opts) + if err != nil { + return nil, err + } + + pluginResource := newResource(p) + tp := trace.NewTracerProvider( + trace.WithBatcher(traceExporter), + trace.WithResource(pluginResource), + ) + + mt := metric.NewMeterProvider( + metric.WithReader(metricReader), + metric.WithResource(pluginResource), + ) + otel.SetErrorHandler(otel.ErrorHandlerFunc(func(err error) { + logger.Debug().Err(err).Msg("otel error") + })) + otel.SetTracerProvider(tp) + otel.SetMeterProvider(mt) + + shutdown = func() { + if err := tp.Shutdown(context.Background()); err != nil { + logger.Error().Err(err).Msg("failed to shutdown OTLP trace exporter") + } + if err := mt.Shutdown(context.Background()); err != nil { + logger.Error().Err(err).Msg("failed to shutdown OTLP metric exporter") + } + } + + return shutdown, nil +} diff --git a/plugin-sdk/serve/opentelemetry_test.go b/plugin-sdk/serve/opentelemetry_test.go new file mode 100644 index 00000000..20b6c119 --- /dev/null +++ b/plugin-sdk/serve/opentelemetry_test.go @@ -0,0 +1,22 @@ +package serve + +import ( + "testing" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/internal/memdb" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/plugin" +) + +func TestNewResource(t *testing.T) { + p := plugin.NewPlugin( + "testPluginV3", + "v1.0.0", + memdb.NewMemDBClient) + + defer func() { + if r := recover(); r != nil { + t.Errorf("failed to instantiate resource: %v", r) + } + }() + newResource(p) +} diff --git a/plugin-sdk/serve/package.go b/plugin-sdk/serve/package.go new file mode 100644 index 00000000..fc835eee --- /dev/null +++ b/plugin-sdk/serve/package.go @@ -0,0 +1,458 @@ +package serve + +import ( + "archive/zip" + "bufio" + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path" + "path/filepath" + "regexp" + "strings" + + cloudquery_api "github.com/cloudquery/cloudquery-api-go" + "github.com/spf13/cobra" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/plugin" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +const ( + pluginPackageShort = "Package plugin for publishing to CloudQuery registry." + pluginPackageLong = `Package plugin for publishing to CloudQuery registry. + +This creates a directory with the plugin binaries, package.json and documentation. +` +) + +// PackageJSON is the package.json file inside the dist directory. It is used by the CloudQuery package command +// to be able to package the plugin with all the needed metadata. +type PackageJSON struct { + SchemaVersion int `json:"schema_version"` + Team string `json:"team"` + Kind plugin.Kind `json:"kind"` + Name string `json:"name"` + Message string `json:"message"` + Version string `json:"version"` + Protocols []int `json:"protocols"` + SupportedTargets []TargetBuild `json:"supported_targets"` + PackageType plugin.PackageType `json:"package_type"` +} + +type TargetBuild struct { + OS string `json:"os"` + Arch string `json:"arch"` + Path string `json:"path"` + Checksum string `json:"checksum"` +} + +func (s *PluginServe) writeTablesJSON(ctx context.Context, dir string) error { + tables, err := s.plugin.Tables(ctx, plugin.TableOptions{ + Tables: []string{"*"}, + }) + if err != nil { + return err + } + flattenedTables := tables.FlattenTables() + tablesToEncode := make([]cloudquery_api.PluginTableCreate, 0, len(flattenedTables)) + for _, t := range flattenedTables { + table := tables.Get(t.Name) + var parent *string + if table.Parent != nil { + parent = &table.Parent.Name + } + relations := make([]string, 0, len(table.Relations)) + if table.Relations != nil { + for _, relation := range table.Relations { + relations = append(relations, relation.Name) + } + } + columns := make([]cloudquery_api.PluginTableColumn, 0, len(table.Columns)) + for _, column := range table.Columns { + columns = append(columns, cloudquery_api.PluginTableColumn{ + Name: column.Name, + Description: column.Description, + Type: column.Type.String(), + IncrementalKey: column.IncrementalKey, + NotNull: column.NotNull, + // PrimaryKey Will be set to true Under the following conditions: + // 1. If the column is a `PrimaryKeyComponent` + // 2. If the column is a `PrimaryKey` and both of the following are true column name is NOT `_cq_id` and there are other columns that are a PrimaryKeyComponent + PrimaryKey: (column.PrimaryKey && !(column.Name == schema.CqIDColumn.Name && len(table.PrimaryKeyComponents()) > 0)) || column.PrimaryKeyComponent, + Unique: column.Unique, + }) + } + tablesToEncode = append(tablesToEncode, cloudquery_api.PluginTableCreate{ + Description: &table.Description, + IsIncremental: &table.IsIncremental, + IsPaid: &table.IsPaid, + Name: table.Name, + Parent: parent, + Relations: &relations, + Title: &table.Title, + Columns: &columns, + }) + } + buffer := &bytes.Buffer{} + m := json.NewEncoder(buffer) + m.SetIndent("", "") + m.SetEscapeHTML(false) + err = m.Encode(tablesToEncode) + if err != nil { + return err + } + outputPath := filepath.Join(dir, "tables.json") + return os.WriteFile(outputPath, buffer.Bytes(), 0o644) +} + +func (s *PluginServe) build(pluginDirectory string, target plugin.BuildTarget, distPath, pluginVersion string) (*TargetBuild, error) { + pluginFileName := fmt.Sprintf("plugin-%s-%s-%s-%s", s.plugin.Name(), pluginVersion, target.OS, target.Arch) + pluginPath := path.Join(distPath, pluginFileName) + importPath, err := s.getModuleName(pluginDirectory) + if err != nil { + return nil, err + } + ldFlags := fmt.Sprintf("-s -w -X %[1]s/plugin.Version=%[2]s -X %[1]s/resources/plugin.Version=%[2]s", importPath, pluginVersion) + args := []string{"build", "-o", pluginPath} + args = append(args, "-buildmode=exe") + args = append(args, "-ldflags", ldFlags) + cmd := exec.Command("go", args...) + cmd.Dir = pluginDirectory + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Env = append(os.Environ(), target.EnvVariables()...) + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("failed to build plugin with `go %v`: %w", args, err) + } + + pluginFile, err := os.Open(pluginPath) + if err != nil { + return nil, fmt.Errorf("failed to open plugin file: %w", err) + } + defer pluginFile.Close() + + zipPluginPath := pluginPath + ".zip" + zipPluginFile, err := os.Create(zipPluginPath) + if err != nil { + return nil, fmt.Errorf("failed to create zip file: %w", err) + } + defer zipPluginFile.Close() + + zipWriter := zip.NewWriter(zipPluginFile) + defer zipWriter.Close() + + pluginZip, err := zipWriter.Create(pluginFileName) + if err != nil { + zipWriter.Close() + return nil, fmt.Errorf("failed to create file in zip archive: %w", err) + } + _, err = io.Copy(pluginZip, pluginFile) + if err != nil { + zipWriter.Close() + return nil, fmt.Errorf("failed to copy plugin file to zip archive: %w", err) + } + err = zipWriter.Close() + if err != nil { + return nil, fmt.Errorf("failed to close zip archive: %w", err) + } + + if err := pluginFile.Close(); err != nil { + return nil, err + } + if err := os.Remove(pluginPath); err != nil { + return nil, fmt.Errorf("failed to remove plugin file: %w", err) + } + + //nolint:govet + targetZip := fmt.Sprintf(pluginFileName + ".zip") + checksum, err := calcChecksum(path.Join(distPath, targetZip)) + if err != nil { + return nil, fmt.Errorf("failed to calculate checksum: %w", err) + } + + return &TargetBuild{ + OS: target.OS, + Arch: target.Arch, + Path: targetZip, + Checksum: "sha256:" + checksum, + }, nil +} + +func calcChecksum(p string) (string, error) { + // calculate SHA-256 checksum + f, err := os.Open(p) + if err != nil { + return "", fmt.Errorf("failed to open file: %w", err) + } + defer f.Close() + hash := sha256.New() + if _, err := io.Copy(hash, f); err != nil { + return "", err + } + return fmt.Sprintf("%x", hash.Sum(nil)), nil +} + +func (*PluginServe) getModuleName(pluginDirectory string) (string, error) { + goMod, err := os.ReadFile(path.Join(pluginDirectory, "go.mod")) + if err != nil { + return "", fmt.Errorf("failed to open go.mod: %w", err) + } + reMod := regexp.MustCompile(`module\s+(.+)\n`) + importPathMatches := reMod.FindStringSubmatch(string(goMod)) + if len(importPathMatches) != 2 { + return "", fmt.Errorf("failed to parse import path from go.mod") + } + importPath := importPathMatches[1] + return strings.TrimSpace(importPath), nil +} + +func (s *PluginServe) writePackageJSON(dir, version, message string, targets []TargetBuild) error { + packageJSON := PackageJSON{ + SchemaVersion: 1, + Name: s.plugin.Name(), + Message: message, + Team: s.plugin.Team(), + Kind: s.plugin.Kind(), + Version: version, + Protocols: s.versions, + SupportedTargets: targets, + PackageType: plugin.PackageTypeNative, + } + buffer := &bytes.Buffer{} + m := json.NewEncoder(buffer) + m.SetIndent("", " ") + m.SetEscapeHTML(false) + err := m.Encode(packageJSON) + if err != nil { + return err + } + outputPath := filepath.Join(dir, "package.json") + return os.WriteFile(outputPath, buffer.Bytes(), 0o644) +} + +func (s *PluginServe) writeSpecJSONSchema(dir string) error { + if s.plugin.JSONSchema() == "" { + return nil + } + + return os.WriteFile(filepath.Join(dir, "spec_json_schema.json"), []byte(s.plugin.JSONSchema()), 0o644) +} + +func (*PluginServe) copyDocs(distPath, docsPath string) error { + err := os.MkdirAll(filepath.Join(distPath, "docs"), 0o755) + if err != nil { + return err + } + dirEntry, err := os.ReadDir(docsPath) + if err != nil { + return err + } + for _, entry := range dirEntry { + if entry.IsDir() { + continue + } + if strings.HasSuffix(entry.Name(), ".md") { + src := filepath.Join(docsPath, entry.Name()) + dst := filepath.Join(distPath, "docs", entry.Name()) + err := copyFile(src, dst) + if err != nil { + return err + } + } + } + return nil +} + +func (*PluginServe) versionRegex() *regexp.Regexp { + return regexp.MustCompile(`^(var)?\s?Version\s*=`) +} + +func (s *PluginServe) validatePluginExports(pluginPath string) error { + st, err := os.Stat(pluginPath) + if err != nil { + return err + } + if !st.IsDir() { + return errors.New("plugin path must be a directory") + } + + checkRelativeDirs := []string{"resources" + string(filepath.Separator) + "plugin", "plugin"} + foundDirs := []string{} + for _, dir := range checkRelativeDirs { + p := filepath.Join(pluginPath, dir) + s, err := os.Stat(p) + if err == nil && s.IsDir() { + foundDirs = append(foundDirs, dir) + } + } + if len(foundDirs) == 0 { + return fmt.Errorf("plugin directory must contain at least one of the following directories: %s", strings.Join(checkRelativeDirs, ", ")) + } + + findVersion := s.versionRegex() + + foundVersion := false + for _, dir := range foundDirs { + p := filepath.Join(pluginPath, dir) + if err := filepath.WalkDir(p, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || foundVersion { + return nil + } + if !strings.HasSuffix(strings.ToLower(d.Name()), ".go") { + return nil + } + + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + if ok, err := containsRegex(f, findVersion); err != nil { + return err + } else if ok { + foundVersion = true + } + return nil + }); err != nil { + return err + } + } + if !foundVersion { + return fmt.Errorf("could not find `Version` global variable in package") + } + + return nil +} + +func copyFile(src, dst string) error { + srcFile, err := os.Open(src) + if err != nil { + return err + } + defer srcFile.Close() + dstFile, err := os.Create(dst) + if err != nil { + return err + } + defer dstFile.Close() + _, err = io.Copy(dstFile, srcFile) + if err != nil { + return err + } + return nil +} + +func containsRegex(r io.Reader, needle *regexp.Regexp) (bool, error) { + scanner := bufio.NewScanner(r) + for scanner.Scan() { + if needle.MatchString(scanner.Text()) { + return true, nil + } + } + return false, scanner.Err() +} + +func (s *PluginServe) newCmdPluginPackage() *cobra.Command { + cmd := &cobra.Command{ + Use: "package -m ", + Short: pluginPackageShort, + Long: pluginPackageLong, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + pluginVersion := args[0] + pluginDirectory := args[1] + distPath := path.Join(pluginDirectory, "dist") + if cmd.Flag("dist-dir").Changed { + distPath = cmd.Flag("dist-dir").Value.String() + } + docsPath := path.Join(pluginDirectory, "docs") + if cmd.Flag("docs-dir").Changed { + docsPath = cmd.Flag("docs-dir").Value.String() + } + message := "" + if !cmd.Flag("message").Changed { + return fmt.Errorf("message is required") + } + message = cmd.Flag("message").Value.String() + if strings.HasPrefix(message, "@") { + messageFile := strings.TrimPrefix(message, "@") + messageBytes, err := os.ReadFile(messageFile) + if err != nil { + return err + } + message = string(messageBytes) + } + message = normalizeMessage(message) + + if err := os.MkdirAll(distPath, 0o755); err != nil { + return err + } + + if s.plugin.Name() == "" { + return fmt.Errorf("plugin name is required for packaging") + } + if s.plugin.Team() == "" { + return fmt.Errorf("plugin team is required (hint: use the plugin.WithTeam() option)") + } + if s.plugin.Kind() == "" { + return fmt.Errorf("plugin kind is required (hint: use the plugin.WithKind() option)") + } + + if err := s.validatePluginExports(pluginDirectory); err != nil { + return err + } + + if s.plugin.Kind() == plugin.KindSource { + if err := s.plugin.Init(cmd.Context(), nil, plugin.NewClientOptions{ + NoConnection: true, + }); err != nil { + return err + } + if err := s.writeTablesJSON(cmd.Context(), distPath); err != nil { + return err + } + } + + targets := []TargetBuild{} + for _, target := range s.plugin.Targets() { + fmt.Println("Building for OS: " + target.OS + ", ARCH: " + target.Arch) + targetBuild, err := s.build(pluginDirectory, target, distPath, pluginVersion) + if err != nil { + return fmt.Errorf("failed to build plugin for %s/%s: %w", target.OS, target.Arch, err) + } + targets = append(targets, *targetBuild) + } + if err := s.writePackageJSON(distPath, pluginVersion, message, targets); err != nil { + return fmt.Errorf("failed to write manifest: %w", err) + } + if err := s.copyDocs(distPath, docsPath); err != nil { + return fmt.Errorf("failed to copy docs: %w", err) + } + if err := s.writeSpecJSONSchema(distPath); err != nil { + return fmt.Errorf("failed to write spec json schema: %w", err) + } + return nil + }, + } + cmd.Flags().StringP("dist-dir", "D", "", "dist directory to output the built plugin. (default: /dist)") + cmd.Flags().StringP("docs-dir", "", "", "docs directory containing markdown files to copy to the dist directory. (default: /docs)") + cmd.Flags().StringP("message", "m", "", "message that summarizes what is new or changed in this version. Use @ to read from file. Supports markdown.") + return cmd +} + +func normalizeMessage(s string) string { + s = strings.TrimSpace(s) + s = strings.ReplaceAll(s, "\r\n", "\n") + s = strings.ReplaceAll(s, "\r", "\n") + return s +} diff --git a/plugin-sdk/serve/package_test.go b/plugin-sdk/serve/package_test.go new file mode 100644 index 00000000..284c33f6 --- /dev/null +++ b/plugin-sdk/serve/package_test.go @@ -0,0 +1,366 @@ +package serve + +/* +import ( + "crypto/sha256" + _ "embed" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/internal/memdb" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/plugin" +) + +//go:embed testdata/memdbtables.json +var memDBPackageJSON string + +//go:embed testdata/source_spec_schema.json +var sourceSpecSchema string + +//go:embed testdata/destination_spec_schema.json +var destinationSpecSchema string + +func TestPluginPackage_Source(t *testing.T) { + _, filename, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("failed to get current file path") + } + dir := filepath.Dir(filepath.Dir(filename)) + simplePluginPath := filepath.Join(dir, "examples/simple_plugin") + packageVersion := "v1.2.3" + p := plugin.NewPlugin( + "test-plugin", + "development", + memdb.NewMemDBClient, + plugin.WithBuildTargets([]plugin.BuildTarget{ + {OS: plugin.GoOSLinux, Arch: plugin.GoArchAmd64}, + {OS: plugin.GoOSWindows, Arch: plugin.GoArchAmd64}, + }), + plugin.WithKind("source"), + plugin.WithTeam("test-team"), + plugin.WithJSONSchema(sourceSpecSchema), + ) + msg := `Test message +with multiple lines and **markdown**` + testCases := []struct { + name string + message string + wantErr bool + }{ + { + name: "inline message", + message: msg, + }, + { + name: "message from file", + message: "@testdata/message.txt", + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("CGO_ENABLED", "0") // disable CGO to ensure we environmental differences don't interfere with the test + srv := Plugin(p) + cmd := srv.newCmdPluginRoot() + distDir := t.TempDir() + cmd.SetArgs([]string{"package", "--dist-dir", distDir, "-m", tc.message, packageVersion, simplePluginPath}) + err := cmd.Execute() + if tc.wantErr && err == nil { + t.Fatalf("expected error, got nil") + } else if err != nil { + t.Fatalf("unexpected error: %v", err) + } + files, err := os.ReadDir(distDir) + if err != nil { + t.Fatal(err) + } + expect := []string{ + "docs", + "package.json", + "plugin-test-plugin-v1.2.3-linux-amd64.zip", + "plugin-test-plugin-v1.2.3-windows-amd64.zip", + "spec_json_schema.json", + "tables.json", + } + if diff := cmp.Diff(expect, fileNames(files)); diff != "" { + t.Fatalf("unexpected files in dist directory (-want +got):\n%s", diff) + } + // expect SHA-256 for the zip files to differ + sha1 := sha256sum(filepath.Join(distDir, "plugin-test-plugin-v1.2.3-linux-amd64.zip")) + sha2 := sha256sum(filepath.Join(distDir, "plugin-test-plugin-v1.2.3-windows-amd64.zip")) + if sha1 == sha2 { + t.Fatalf("expected SHA-256 for linux and windows zip files to differ, but they are the same: %s", sha1) + } + + expectPackage := PackageJSON{ + SchemaVersion: 1, + Name: "test-plugin", + Team: "test-team", + Kind: "source", + Message: msg, + Version: packageVersion, + Protocols: []int{3}, + SupportedTargets: []TargetBuild{ + {OS: plugin.GoOSLinux, Arch: plugin.GoArchAmd64, Path: "plugin-test-plugin-v1.2.3-linux-amd64.zip", Checksum: "sha256:" + sha256sum(filepath.Join(distDir, "plugin-test-plugin-v1.2.3-linux-amd64.zip"))}, + {OS: plugin.GoOSWindows, Arch: plugin.GoArchAmd64, Path: "plugin-test-plugin-v1.2.3-windows-amd64.zip", Checksum: "sha256:" + sha256sum(filepath.Join(distDir, "plugin-test-plugin-v1.2.3-windows-amd64.zip"))}, + }, + PackageType: plugin.PackageTypeNative, + } + checkPackageJSONContents(t, filepath.Join(distDir, "package.json"), expectPackage) + + expectDocs := []string{ + "configuration.md", + "overview.md", + } + checkDocs(t, filepath.Join(distDir, "docs"), expectDocs) + checkTables(t, distDir) + checkFileContent(t, filepath.Join(distDir, "spec_json_schema.json"), sourceSpecSchema) + }) + } +} + +func TestPluginPackage_Destination(t *testing.T) { + _, filename, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("failed to get current file path") + } + dir := filepath.Dir(filepath.Dir(filename)) + simplePluginPath := filepath.Join(dir, "examples/simple_plugin") + packageVersion := "v1.2.3" + p := plugin.NewPlugin( + "test-plugin", + "development", + memdb.NewMemDBClient, + plugin.WithBuildTargets([]plugin.BuildTarget{ + {OS: plugin.GoOSWindows, Arch: plugin.GoArchAmd64}, + {OS: plugin.GoOSDarwin, Arch: plugin.GoArchAmd64}, + }), + plugin.WithKind("destination"), + plugin.WithTeam("test-team"), + plugin.WithJSONSchema(destinationSpecSchema), + ) + msg := `Test message +with multiple lines and **markdown**` + testCases := []struct { + name string + message string + wantErr bool + }{ + { + name: "inline message", + message: msg, + }, + { + name: "message from file", + message: "@testdata/message.txt", + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("CGO_ENABLED", "0") // disable CGO to ensure we environmental differences don't interfere with the test + srv := Plugin(p) + cmd := srv.newCmdPluginRoot() + distDir := t.TempDir() + cmd.SetArgs([]string{"package", "--dist-dir", distDir, "-m", tc.message, packageVersion, simplePluginPath}) + err := cmd.Execute() + if tc.wantErr && err == nil { + t.Fatalf("expected error, got nil") + } else if err != nil { + t.Fatalf("unexpected error: %v", err) + } + files, err := os.ReadDir(distDir) + if err != nil { + t.Fatal(err) + } + expect := []string{ + "docs", + "package.json", + "plugin-test-plugin-v1.2.3-darwin-amd64.zip", + "plugin-test-plugin-v1.2.3-windows-amd64.zip", + "spec_json_schema.json", + } + if diff := cmp.Diff(expect, fileNames(files)); diff != "" { + t.Fatalf("unexpected files in dist directory (-want +got):\n%s", diff) + } + // expect SHA-256 for the zip files to differ + sha1 := sha256sum(filepath.Join(distDir, "plugin-test-plugin-v1.2.3-windows-amd64.zip")) + sha2 := sha256sum(filepath.Join(distDir, "plugin-test-plugin-v1.2.3-darwin-amd64.zip")) + if sha1 == sha2 { + t.Fatalf("expected SHA-256 for windows and darwin zip files to differ, but they are the same: %s", sha1) + } + + expectPackage := PackageJSON{ + SchemaVersion: 1, + Team: "test-team", + Kind: "destination", + Name: "test-plugin", + Message: msg, + Version: "v1.2.3", + Protocols: []int{3}, + SupportedTargets: []TargetBuild{ + {OS: plugin.GoOSWindows, Arch: plugin.GoArchAmd64, Path: "plugin-test-plugin-v1.2.3-windows-amd64.zip", Checksum: "sha256:" + sha256sum(filepath.Join(distDir, "plugin-test-plugin-v1.2.3-windows-amd64.zip"))}, + {OS: plugin.GoOSDarwin, Arch: plugin.GoArchAmd64, Path: "plugin-test-plugin-v1.2.3-darwin-amd64.zip", Checksum: "sha256:" + sha256sum(filepath.Join(distDir, "plugin-test-plugin-v1.2.3-darwin-amd64.zip"))}, + }, + PackageType: plugin.PackageTypeNative, + } + checkPackageJSONContents(t, filepath.Join(distDir, "package.json"), expectPackage) + + expectDocs := []string{ + "configuration.md", + "overview.md", + } + checkDocs(t, filepath.Join(distDir, "docs"), expectDocs) + checkFileContent(t, filepath.Join(distDir, "spec_json_schema.json"), destinationSpecSchema) + }) + } +} + +func TestVersionRegex(t *testing.T) { + t.Parallel() + re := Plugin(&plugin.Plugin{}).versionRegex() + testCases := []struct { + input string + want bool + }{ + { + input: `var Version = ""`, + want: true, + }, + { + input: `var ( +Version = "" +) +`, + want: true, + }, + { + input: ` var Version = ""`, + want: false, + }, + { + input: `var ( + Version = "" +) +`, + }, + { + input: `var ( + Version = "" +) +`, + want: true, + }, + } + for i, tc := range testCases { + tc := tc + t.Run(fmt.Sprintf("Case %d", i+1), func(t *testing.T) { + t.Parallel() + // only match the line with "Version" + lines := strings.Split(tc.input, "\n") + realInput := "" + for _, line := range lines { + if strings.Contains(line, "Version") { + realInput = line + break + } + } + if realInput == "" { + t.Fatalf("failed to find line with Version: %q", tc.input) + } + got := re.MatchString(realInput) + require.Equalf(t, tc.want, got, "input: %q", realInput) + }) + } +} + +func sha256sum(filename string) string { + f, err := os.Open(filename) + if err != nil { + panic(err) + } + defer f.Close() + h := sha256.New() + _, err = io.Copy(h, f) + if err != nil { + panic(err) + } + return fmt.Sprintf("%x", h.Sum(nil)) +} + +func checkDocs(t *testing.T, dir string, expect []string) { + files, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(expect, fileNames(files)); diff != "" { + t.Fatalf("unexpected files in docs directory (-want +got):\n%s", diff) + } +} + +func checkTables(t *testing.T, distDir string) { + content, err := os.ReadFile(filepath.Join(distDir, "tables.json")) + if err != nil { + t.Fatal(err) + } + var actual any + err = json.Unmarshal(content, &actual) + require.NoError(t, err) + + var expected any + err = json.Unmarshal([]byte(memDBPackageJSON), &expected) + require.NoError(t, err) + + require.Equal(t, expected, actual, "tables.json content mismatch") +} + +func checkPackageJSONContents(t *testing.T, filename string, expect PackageJSON) { + f, err := os.Open(filename) + if err != nil { + t.Fatalf("failed to open package.json: %v", err) + } + defer f.Close() + b, err := io.ReadAll(f) + if err != nil { + t.Fatalf("failed to read package.json: %v", err) + } + j := PackageJSON{} + err = json.Unmarshal(b, &j) + if err != nil { + t.Fatalf("failed to unmarshal package.json: %v", err) + } + if diff := cmp.Diff(expect, j); diff != "" { + t.Fatalf("package.json contents mismatch (-want +got):\n%s", diff) + } +} + +func checkFileContent(t *testing.T, filename string, expect string) { + f, err := os.Open(filename) + if err != nil { + t.Fatalf("failed to open %s: %v", filename, err) + } + defer f.Close() + b, err := io.ReadAll(f) + if err != nil { + t.Fatalf("failed to read %s: %v", filename, err) + } + if diff := cmp.Diff(expect, string(b)); diff != "" { + t.Fatalf("%s contents mismatch (-want +got):\n%s", filename, diff) + } +} + +func fileNames(files []os.DirEntry) []string { + names := make([]string, 0, len(files)) + for _, file := range files { + names = append(names, file.Name()) + } + return names +} +*/ diff --git a/plugin-sdk/serve/plugin.go b/plugin-sdk/serve/plugin.go new file mode 100644 index 00000000..5cdd5357 --- /dev/null +++ b/plugin-sdk/serve/plugin.go @@ -0,0 +1,255 @@ +package serve + +import ( + "context" + "fmt" + "net" + "os" + "os/signal" + "strings" + "syscall" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/helpers/grpczerolog" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/plugin" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/premium" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" + + pbDestinationV0 "github.com/cloudquery/plugin-pb-go/pb/destination/v0" + pbDestinationV1 "github.com/cloudquery/plugin-pb-go/pb/destination/v1" + pbdiscoveryv0 "github.com/cloudquery/plugin-pb-go/pb/discovery/v0" + pbdiscoveryv1 "github.com/cloudquery/plugin-pb-go/pb/discovery/v1" + pbv3 "github.com/cloudquery/plugin-pb-go/pb/plugin/v3" + + discoveryServerV0 "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/internal/servers/discovery/v0" + discoveryServerV1 "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/internal/servers/discovery/v1" + + "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + "google.golang.org/grpc" + "google.golang.org/grpc/test/bufconn" + + serverDestinationV0 "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/internal/servers/destination/v0" + serverDestinationV1 "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/internal/servers/destination/v1" + serversv3 "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/internal/servers/plugin/v3" +) + +type PluginServe struct { + plugin *plugin.Plugin + args []string + destinationV0V1Server bool + testListener bool + testListenerConn *bufconn.Listener + versions []int +} + +type PluginOption func(*PluginServe) + +// WithDestinationV0V1Server is used to include destination v0 and v1 server to work +// with older sources +func WithDestinationV0V1Server() PluginOption { + return func(s *PluginServe) { + s.destinationV0V1Server = true + } +} + +// WithArgs used to serve the plugin with predefined args instead of os.Args +func WithArgs(args ...string) PluginOption { + return func(s *PluginServe) { + s.args = args + } +} + +// WithTestListener means that the plugin will be served with an in-memory listener +// available via testListener() method instead of a network listener. +func WithTestListener() PluginOption { + return func(s *PluginServe) { + s.testListener = true + s.testListenerConn = bufconn.Listen(testBufSize) + } +} + +const servePluginShort = `Start plugin server` + +func Plugin(p *plugin.Plugin, opts ...PluginOption) *PluginServe { + s := &PluginServe{ + plugin: p, + versions: []int{3}, + } + for _, opt := range opts { + opt(s) + } + return s +} + +func (s *PluginServe) bufPluginDialer(context.Context, string) (net.Conn, error) { + return s.testListenerConn.Dial() +} + +func (s *PluginServe) Serve(ctx context.Context) error { + if err := types.RegisterAllExtensions(); err != nil { + return err + } + defer func() { + if err := types.UnregisterAllExtensions(); err != nil { + log.Error().Err(err).Msg("failed to unregister all extensions") + } + }() + cmd := s.newCmdPluginRoot() + if s.args != nil { + cmd.SetArgs(s.args) + } + return cmd.ExecuteContext(ctx) +} + +func (s *PluginServe) newCmdPluginServe() *cobra.Command { + var address string + var network string + var noSentry bool + var otelEndpoint string + var otelEndpointHeaders []string + var otelEndpointInsecure bool + var otelEndpointURLPath string + var licenseFile string + logLevel := newEnum([]string{"trace", "debug", "info", "warn", "error"}, "info") + logFormat := newEnum([]string{"text", "json"}, "text") + telemetryLevel := newEnum([]string{"none", "errors", "stats", "all"}, "all") + err := telemetryLevel.Set(getEnvOrDefault("CQ_TELEMETRY_LEVEL", telemetryLevel.Value)) + if err != nil { + //nolint:govet + fmt.Fprintf(os.Stderr, "failed to set telemetry level: "+err.Error()) + os.Exit(1) + } + + cmd := &cobra.Command{ + Use: "serve", + Short: servePluginShort, + Long: servePluginShort, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + zerologLevel, err := zerolog.ParseLevel(logLevel.String()) + if err != nil { + return err + } + var logger zerolog.Logger + if logFormat.String() == "json" { + logger = zerolog.New(os.Stdout).Level(zerologLevel) + } else { + logger = log.Output(zerolog.ConsoleWriter{Out: os.Stdout}).Level(zerologLevel) + } + + shutdown, err := setupOtel(cmd.Context(), logger, s.plugin, otelEndpoint, otelEndpointInsecure, otelEndpointHeaders, otelEndpointURLPath) + if err != nil { + return fmt.Errorf("failed to setup OpenTelemetry: %w", err) + } + defer shutdown() + + if licenseFile != "" { + switch err := premium.ValidateLicense(logger, s.plugin.Meta(), licenseFile); err { + case nil: + s.plugin.SetSkipUsageClient(true) + case premium.ErrLicenseNotApplicable: + // no-op: Treat as if no license was provided + default: + return fmt.Errorf("failed to validate license: %w", err) + } + } + + var listener net.Listener + if s.testListener { + listener = s.testListenerConn + } else { + listener, err = net.Listen(network, address) + if err != nil { + return fmt.Errorf("failed to listen %s:%s: %w", network, address, err) + } + } + defer listener.Close() + // source plugins can only accept one connection at a time + // unlike destination plugins that can accept multiple connections + // limitListener := netutil.LimitListener(listener, 1) + // See logging pattern https://github.com/grpc-ecosystem/go-grpc-middleware/blob/v2/providers/zerolog/examples_test.go + grpcServer := grpc.NewServer( + grpc.ChainUnaryInterceptor( + logging.UnaryServerInterceptor(grpczerolog.InterceptorLogger(logger)), + ), + grpc.ChainStreamInterceptor( + logging.StreamServerInterceptor(grpczerolog.InterceptorLogger(logger)), + ), + grpc.MaxRecvMsgSize(MaxMsgSize), + grpc.MaxSendMsgSize(MaxMsgSize), + ) + s.plugin.SetLogger(logger) + pbv3.RegisterPluginServer(grpcServer, &serversv3.Server{ + Plugin: s.plugin, + Logger: logger, + }) + if s.destinationV0V1Server { + pbDestinationV1.RegisterDestinationServer(grpcServer, &serverDestinationV1.Server{ + Plugin: s.plugin, + Logger: logger, + }) + pbDestinationV0.RegisterDestinationServer(grpcServer, &serverDestinationV0.Server{ + Plugin: s.plugin, + Logger: logger, + }) + } + pbdiscoveryv0.RegisterDiscoveryServer(grpcServer, &discoveryServerV0.Server{ + Versions: []string{"v0", "v1", "v2", "v3"}, + }) + pbdiscoveryv1.RegisterDiscoveryServer(grpcServer, &discoveryServerV1.Server{ + Versions: []int32{0, 1, 2, 3}, + }) + + ctx := cmd.Context() + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + defer func() { + signal.Stop(c) + }() + + go func() { + select { + case sig := <-c: + logger.Info().Str("address", listener.Addr().String()).Str("signal", sig.String()).Msg("Got stop signal. Plugin server shutting down") + grpcServer.Stop() + case <-ctx.Done(): + logger.Info().Str("address", listener.Addr().String()).Msg("Context cancelled. Plugin server shutting down") + grpcServer.Stop() + } + }() + + logger.Info().Str("address", listener.Addr().String()).Msg("Plugin server listening") + if err := grpcServer.Serve(listener); err != nil { + return fmt.Errorf("failed to serve: %w", err) + } + return nil + }, + } + cmd.Flags().StringVar(&address, "address", "localhost:7777", "address to serve on. can be tcp: `localhost:7777` or unix socket: `/tmp/plugin.rpc.sock`") + cmd.Flags().StringVar(&network, "network", "tcp", `the network must be "tcp", "tcp4", "tcp6", "unix" or "unixpacket"`) + cmd.Flags().Var(logLevel, "log-level", fmt.Sprintf("log level. one of: %s", strings.Join(logLevel.Allowed, ","))) + cmd.Flags().Var(logFormat, "log-format", fmt.Sprintf("log format. one of: %s", strings.Join(logFormat.Allowed, ","))) + cmd.Flags().StringVar(&otelEndpoint, "otel-endpoint", "", "Open Telemetry HTTP collector endpoint") + cmd.Flags().StringVar(&otelEndpointURLPath, "otel-endpoint-urlpath", "", "Open Telemetry HTTP collector endpoint URL path") + cmd.Flags().StringArrayVar(&otelEndpointHeaders, "otel-endpoint-headers", []string{}, "Open Telemetry HTTP collector endpoint headers") + cmd.Flags().BoolVar(&otelEndpointInsecure, "otel-endpoint-insecure", false, "use Open Telemetry HTTP endpoint (for development only)") + cmd.Flags().BoolVar(&noSentry, "no-sentry", false, "disable sentry") + cmd.Flags().StringVar(&licenseFile, "license", "", "Path to offline license file or directory") + + return cmd +} + +func (s *PluginServe) newCmdPluginRoot() *cobra.Command { + cmd := &cobra.Command{ + Use: fmt.Sprintf("%s ", s.plugin.Name()), + } + cmd.AddCommand(s.newCmdPluginServe()) + cmd.AddCommand(s.newCmdPluginDoc()) + cmd.AddCommand(s.newCmdPluginPackage()) + cmd.AddCommand(s.newCmdPluginInfo()) + cmd.CompletionOptions.DisableDefaultCmd = true + cmd.Version = s.plugin.Version() + return cmd +} diff --git a/plugin-sdk/serve/plugin_test.go b/plugin-sdk/serve/plugin_test.go new file mode 100644 index 00000000..b7e36e81 --- /dev/null +++ b/plugin-sdk/serve/plugin_test.go @@ -0,0 +1,189 @@ +package serve + +import ( + "bytes" + "context" + "io" + "sync" + "testing" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/ipc" + "github.com/apache/arrow/go/v16/arrow/memory" + pb "github.com/cloudquery/plugin-pb-go/pb/plugin/v3" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/internal/memdb" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/plugin" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +func TestPluginServe(t *testing.T) { + p := plugin.NewPlugin( + "testPluginV3", + "v1.0.0", + memdb.NewMemDBClient) + srv := Plugin(p, WithArgs("serve"), WithTestListener()) + ctx := context.Background() + ctx, cancel := context.WithCancel(ctx) + var wg sync.WaitGroup + wg.Add(1) + var serverErr error + go func() { + defer wg.Done() + serverErr = srv.Serve(ctx) + }() + defer func() { + cancel() + wg.Wait() + }() + + // https://stackoverflow.com/questions/42102496/testing-a-grpc-service + // TODO: Remove once there's a documented migration path per https://github.com/grpc/grpc-go/issues/7244 + // nolint:staticcheck + conn, err := grpc.DialContext(ctx, "bufnet", grpc.WithContextDialer(srv.bufPluginDialer), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithBlock()) + if err != nil { + t.Fatalf("Failed to dial bufnet: %v", err) + } + + c := pb.NewPluginClient(conn) + + getNameRes, err := c.GetName(ctx, &pb.GetName_Request{}) + if err != nil { + t.Fatal(err) + } + if getNameRes.Name != "testPluginV3" { + t.Fatalf("expected name to be testPluginV3 but got %s", getNameRes.Name) + } + + getVersionResponse, err := c.GetVersion(ctx, &pb.GetVersion_Request{}) + if err != nil { + t.Fatal(err) + } + if getVersionResponse.Version != "v1.0.0" { + t.Fatalf("Expected version to be v1.0.0 but got %s", getVersionResponse.Version) + } + + if _, err := c.Init(ctx, &pb.Init_Request{}); err != nil { + t.Fatal(err) + } + + getTablesRes, err := c.GetTables(ctx, &pb.GetTables_Request{Tables: []string{"*"}}) + if err != nil { + t.Fatal(err) + } + schemas, err := pb.NewSchemasFromBytes(getTablesRes.Tables) + if err != nil { + t.Fatal(err) + } + tables, err := schema.NewTablesFromArrowSchemas(schemas) + if err != nil { + t.Fatal(err) + } + + if len(tables) != 3 { + t.Fatalf("Expected 2 tables but got %d", len(tables)) + } + testTable := schema.Table{ + Name: "test_table", + Columns: []schema.Column{ + { + Name: "col1", + Type: arrow.BinaryTypes.String, + }, + }, + } + bldr := array.NewRecordBuilder(memory.DefaultAllocator, testTable.ToArrowSchema()) + bldr.Field(0).(*array.StringBuilder).Append("test") + record := bldr.NewRecord() + + recordBytes, err := pb.RecordToBytes(record) + if err != nil { + t.Fatal(err) + } + sc := testTable.ToArrowSchema() + tableBytes, err := pb.SchemaToBytes(sc) + if err != nil { + t.Fatal(err) + } + writeClient, err := c.Write(ctx) + if err != nil { + t.Fatal(err) + } + + if err := writeClient.Send(&pb.Write_Request{ + Message: &pb.Write_Request_MigrateTable{ + MigrateTable: &pb.Write_MessageMigrateTable{ + Table: tableBytes, + }, + }, + }); err != nil { + t.Fatal(err) + } + if err := writeClient.Send(&pb.Write_Request{ + Message: &pb.Write_Request_Insert{ + Insert: &pb.Write_MessageInsert{ + Record: recordBytes, + }, + }, + }); err != nil { + t.Fatal(err) + } + if _, err := writeClient.CloseAndRecv(); err != nil { + t.Fatal(err) + } + + syncClient, err := c.Sync(ctx, &pb.Sync_Request{ + Tables: []string{"test_table"}, + }) + if err != nil { + t.Fatal(err) + } + var resources []arrow.Record + for { + r, err := syncClient.Recv() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + m := r.Message.(*pb.Sync_Response_Insert) + rdr, err := ipc.NewReader(bytes.NewReader(m.Insert.Record)) + if err != nil { + t.Fatal(err) + } + for rdr.Next() { + rec := rdr.Record() + rec.Retain() + resources = append(resources, rec) + } + } + + totalResources := 0 + for _, resource := range resources { + sc := resource.Schema() + tableName, ok := sc.Metadata().GetValue(schema.MetadataTableName) + if !ok { + t.Fatal("Expected table name metadata to be set") + } + if tableName != "test_table" { + t.Fatalf("Expected resource with table name test_table. got: %s", tableName) + } + if len(resource.Columns()) != 1 { + t.Fatalf("Expected resource with data length 1 but got %d", len(resource.Columns())) + } + totalResources++ + } + if totalResources != 1 { + t.Fatalf("Expected 1 resource on channel but got %d", totalResources) + } + + cancel() + wg.Wait() + if serverErr != nil { + t.Fatal(serverErr) + } +} diff --git a/plugin-sdk/serve/state_test.go b/plugin-sdk/serve/state_test.go new file mode 100644 index 00000000..9b9a3e89 --- /dev/null +++ b/plugin-sdk/serve/state_test.go @@ -0,0 +1,159 @@ +package serve + +import ( + "context" + "sync" + "testing" + + pb "github.com/cloudquery/plugin-pb-go/pb/plugin/v3" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/internal/clients/state/v3" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/internal/memdb" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/plugin" +) + +func TestState(t *testing.T) { + p := plugin.NewPlugin( + "testPluginV3", + "v1.0.0", + memdb.NewMemDBClient) + srv := Plugin(p, WithArgs("serve"), WithTestListener()) + ctx := context.Background() + ctx, cancel := context.WithCancel(ctx) + var wg sync.WaitGroup + wg.Add(1) + var serverErr error + go func() { + defer wg.Done() + serverErr = srv.Serve(ctx) + }() + defer func() { + cancel() + wg.Wait() + }() + + // https://stackoverflow.com/questions/42102496/testing-a-grpc-service + // TODO: Remove once there's a documented migration path per https://github.com/grpc/grpc-go/issues/7244 + // nolint:staticcheck + conn, err := grpc.DialContext(ctx, "bufnet", grpc.WithContextDialer(srv.bufPluginDialer), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithBlock()) + if err != nil { + t.Fatalf("Failed to dial bufnet: %v", err) + } + + c := pb.NewPluginClient(conn) + if _, err := c.Init(ctx, &pb.Init_Request{}); err != nil { + t.Fatal(err) + } + stateClient, err := state.NewClient(ctx, conn, "test") + if err != nil { + t.Fatal(err) + } + + if err := stateClient.SetKey(ctx, "key", "value"); err != nil { + t.Fatal(err) + } + + val, err := stateClient.GetKey(ctx, "key") + if err != nil { + t.Fatal(err) + } + if val != "value" { + t.Fatalf("expected value to be value but got %s", val) + } + + if err := stateClient.Flush(ctx); err != nil { + t.Fatal(err) + } + stateClient, err = state.NewClient(ctx, conn, "test") + if err != nil { + t.Fatal(err) + } + val, err = stateClient.GetKey(ctx, "key") + if err != nil { + t.Fatal(err) + } + if val != "value" { + t.Fatalf("expected value to be value but got %s", val) + } + + cancel() + wg.Wait() + if serverErr != nil { + t.Fatal(serverErr) + } +} + +func TestStateOverwriteGetLatest(t *testing.T) { + p := plugin.NewPlugin( + "testPluginV3", + "v1.0.0", + memdb.NewMemDBClient) + srv := Plugin(p, WithArgs("serve"), WithTestListener()) + ctx := context.Background() + ctx, cancel := context.WithCancel(ctx) + var wg sync.WaitGroup + wg.Add(1) + var serverErr error + go func() { + defer wg.Done() + serverErr = srv.Serve(ctx) + }() + defer func() { + cancel() + wg.Wait() + }() + + // https://stackoverflow.com/questions/42102496/testing-a-grpc-service + // TODO: Remove once there's a documented migration path per https://github.com/grpc/grpc-go/issues/7244 + // nolint:staticcheck + conn, err := grpc.DialContext(ctx, "bufnet", grpc.WithContextDialer(srv.bufPluginDialer), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithBlock()) + if err != nil { + t.Fatalf("Failed to dial bufnet: %v", err) + } + + c := pb.NewPluginClient(conn) + if _, err := c.Init(ctx, &pb.Init_Request{}); err != nil { + t.Fatal(err) + } + + table := state.VersionedTable("test_no_pk") + // Remove PKs + for i := range table.Columns { + table.Columns[i].PrimaryKey = false + } + + stateClient, err := state.NewClientWithTable(ctx, conn, table) + if err != nil { + t.Fatal(err) + } + + for _, v := range []string{"valua", "value1", "value3", "value2"} { + if err := stateClient.SetKey(ctx, "key", v); err != nil { + t.Fatal(err) + } + // Without Flush(), value will only be updated in memory, and we won't get duplicate entries in memdb + if err := stateClient.Flush(ctx); err != nil { + t.Fatal(err) + } + } + + stateClient, err = state.NewClientWithTable(ctx, conn, table) + if err != nil { + t.Fatal(err) + } + val, err := stateClient.GetKey(ctx, "key") + if err != nil { + t.Fatal(err) + } + if val != `value2` { + t.Fatalf("expected value to be %q but got %q", `value2`, val) + } + + cancel() + wg.Wait() + if serverErr != nil { + t.Fatal(serverErr) + } +} diff --git a/plugin-sdk/serve/testdata/destination_spec_schema.json b/plugin-sdk/serve/testdata/destination_spec_schema.json new file mode 100644 index 00000000..2396d773 --- /dev/null +++ b/plugin-sdk/serve/testdata/destination_spec_schema.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "#/$defs/Spec", + "$defs": { + "Spec": { + "properties": { + "connection_string": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + } + } + } \ No newline at end of file diff --git a/plugin-sdk/serve/testdata/memdbtables.json b/plugin-sdk/serve/testdata/memdbtables.json new file mode 100644 index 00000000..bf27718c --- /dev/null +++ b/plugin-sdk/serve/testdata/memdbtables.json @@ -0,0 +1,61 @@ +[ + { + "columns": [ + { + "description": "col1 description", + "incremental_key": false, + "name": "col1", + "not_null": true, + "primary_key": true, + "type": "int64", + "unique": true + } + ], + "description": "", + "is_incremental": false, + "name": "table1", + "relations": ["table2"], + "title": "", + "is_paid": false + }, + { + "columns": [ + { + "description": "col1 description", + "incremental_key": true, + "name": "col1", + "not_null": false, + "primary_key": false, + "type": "uuid", + "unique": false + } + ], + "description": "", + "parent": "table1", + "relations": ["table3"], + "is_incremental": false, + "name": "table2", + "title": "", + "is_paid": false + }, + { + "columns": [ + { + "description": "col1 description", + "incremental_key": true, + "name": "col1", + "not_null": false, + "primary_key": false, + "type": "uuid", + "unique": false + } + ], + "description": "", + "parent": "table2", + "relations": [], + "is_incremental": false, + "name": "table3", + "title": "", + "is_paid": true + } +] diff --git a/plugin-sdk/serve/testdata/message.txt b/plugin-sdk/serve/testdata/message.txt new file mode 100644 index 00000000..9dc1144e --- /dev/null +++ b/plugin-sdk/serve/testdata/message.txt @@ -0,0 +1,2 @@ +Test message +with multiple lines and **markdown** \ No newline at end of file diff --git a/plugin-sdk/serve/testdata/source_spec_schema.json b/plugin-sdk/serve/testdata/source_spec_schema.json new file mode 100644 index 00000000..e2266bf6 --- /dev/null +++ b/plugin-sdk/serve/testdata/source_spec_schema.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "#/$defs/Spec", + "$defs": { + "Spec": { + "properties": { + "concurrency": { + "type": "integer" + } + }, + "additionalProperties": false, + "type": "object" + } + } + } \ No newline at end of file diff --git a/plugin-sdk/state/client.go b/plugin-sdk/state/client.go new file mode 100644 index 00000000..faf8d3ff --- /dev/null +++ b/plugin-sdk/state/client.go @@ -0,0 +1,109 @@ +package state + +import ( + "context" + "fmt" + "slices" + + pbDiscovery "github.com/cloudquery/plugin-pb-go/pb/discovery/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + + stateV3 "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/internal/clients/state/v3" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/plugin" +) + +const defaultMaxMsgSizeInBytes = 100 * 1024 * 1024 // 100 MiB + +type Client interface { + SetKey(ctx context.Context, key string, value string) error + GetKey(ctx context.Context, key string) (string, error) + Flush(ctx context.Context) error + Close() error +} + +type ClientOptions struct { + Versioned bool +} + +type ConnectionOptions struct { + MaxMsgSizeInBytes int +} + +func NewClient(ctx context.Context, conn *grpc.ClientConn, tableName string) (Client, error) { + return NewClientWithOptions(ctx, conn, tableName, ClientOptions{}) +} + +func NewClientWithOptions(ctx context.Context, conn *grpc.ClientConn, tableName string, opts ClientOptions) (Client, error) { + discoveryClient := pbDiscovery.NewDiscoveryClient(conn) + versions, err := discoveryClient.GetVersions(ctx, &pbDiscovery.GetVersions_Request{}) + if err != nil { + return nil, err + } + if !slices.Contains(versions.Versions, 3) { + return nil, fmt.Errorf("please upgrade your state backend plugin. state supporting version 3 plugin has %v", versions.Versions) + } + + if opts.Versioned { + return stateV3.NewClientWithTable(ctx, conn, stateV3.VersionedTable(tableName)) + } + return stateV3.NewClient(ctx, conn, tableName) +} + +// NewConnectedClient returns a state client and initialises the gRPC connection to the state backend with a 100MiB max message size. +// The state client is guaranteed to be non-nil (it defaults to the NoOpClient). +// You must call Close() on the returned Client object. +func NewConnectedClient(ctx context.Context, backendOpts *plugin.BackendOptions) (Client, error) { + return NewConnectedClientWithOptions(ctx, backendOpts, ConnectionOptions{MaxMsgSizeInBytes: defaultMaxMsgSizeInBytes}, ClientOptions{}) +} + +// NewConnectedClientWithOptions returns a state client and initialises the gRPC connection to the state backend. +// The state client is guaranteed to be non-nil (it defaults to the NoOpClient). +// You must call Close() on the returned Client object. +func NewConnectedClientWithOptions(ctx context.Context, backendOpts *plugin.BackendOptions, connOpts ConnectionOptions, clOpts ClientOptions) (Client, error) { + if backendOpts == nil { + return &NoOpClient{}, nil + } + + // TODO: Remove once there's a documented migration path per https://github.com/grpc/grpc-go/issues/7244 + // nolint:staticcheck + backendConn, err := grpc.DialContext(ctx, backendOpts.Connection, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultCallOptions( + grpc.MaxCallRecvMsgSize(connOpts.MaxMsgSizeInBytes), + grpc.MaxCallSendMsgSize(connOpts.MaxMsgSizeInBytes), + ), + ) + + if err != nil { + return &NoOpClient{}, fmt.Errorf("failed to dial grpc source plugin at %s: %w", backendOpts.Connection, err) + } + + stateClient, err := NewClientWithOptions(ctx, backendConn, backendOpts.TableName, clOpts) + if err != nil { + return &NoOpClient{}, fmt.Errorf("failed to create state client: %w", err) + } + + return stateClient, nil +} + +type NoOpClient struct{} + +func (*NoOpClient) SetKey(_ context.Context, _ string, _ string) error { + return nil +} + +func (*NoOpClient) GetKey(_ context.Context, _ string) (string, error) { + return "", nil +} + +func (*NoOpClient) Flush(_ context.Context) error { + return nil +} + +func (*NoOpClient) Close() error { + return nil +} + +// static check +var _ Client = (*NoOpClient)(nil) diff --git a/plugin-sdk/transformers/ignore_in_tests.go b/plugin-sdk/transformers/ignore_in_tests.go new file mode 100644 index 00000000..16c551a0 --- /dev/null +++ b/plugin-sdk/transformers/ignore_in_tests.go @@ -0,0 +1,11 @@ +package transformers + +import "reflect" + +type IgnoreInTestsTransformer func(field reflect.StructField) bool + +func DefaultIgnoreInTestsTransformer(_ reflect.StructField) bool { + return false +} + +var _ IgnoreInTestsTransformer = DefaultIgnoreInTestsTransformer diff --git a/plugin-sdk/transformers/name.go b/plugin-sdk/transformers/name.go new file mode 100644 index 00000000..c4a13542 --- /dev/null +++ b/plugin-sdk/transformers/name.go @@ -0,0 +1,29 @@ +package transformers + +import ( + "reflect" + "strings" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/caser" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +type NameTransformer func(reflect.StructField) (string, error) + +var defaultCaser = caser.New() + +func DefaultNameTransformer(field reflect.StructField) (string, error) { + name := field.Name + if jsonTag := strings.Split(field.Tag.Get("json"), ",")[0]; len(jsonTag) > 0 { + // return empty string if the field is not related api response + if jsonTag == "-" { + return "", nil + } + if nameFromJSONTag := defaultCaser.ToSnake(jsonTag); schema.ValidColumnName(nameFromJSONTag) { + return nameFromJSONTag, nil + } + } + return defaultCaser.ToSnake(name), nil +} + +var _ NameTransformer = DefaultNameTransformer diff --git a/plugin-sdk/transformers/options.go b/plugin-sdk/transformers/options.go new file mode 100644 index 00000000..d5ee0952 --- /dev/null +++ b/plugin-sdk/transformers/options.go @@ -0,0 +1,70 @@ +package transformers + +type StructTransformerOption func(*structTransformer) + +// WithUnwrapAllEmbeddedStructs instructs codegen to unwrap all embedded fields (1 level deep only) +func WithUnwrapAllEmbeddedStructs() StructTransformerOption { + return func(t *structTransformer) { + t.unwrapAllEmbeddedStructFields = true + } +} + +// WithUnwrapStructFields allows to unwrap specific struct fields (1 level deep only) +func WithUnwrapStructFields(fields ...string) StructTransformerOption { + return func(t *structTransformer) { + t.structFieldsToUnwrap = fields + } +} + +// WithSkipFields allows to specify what struct fields should be skipped. +func WithSkipFields(fields ...string) StructTransformerOption { + return func(t *structTransformer) { + t.skipFields = fields + } +} + +// WithNameTransformer overrides how column name will be determined. +// DefaultNameTransformer is used as the default. +func WithNameTransformer(transformer NameTransformer) StructTransformerOption { + return func(t *structTransformer) { + t.nameTransformer = transformer + } +} + +// WithTypeTransformer overrides how column type will be determined. +// DefaultTypeTransformer is used as the default. +func WithTypeTransformer(transformer TypeTransformer) StructTransformerOption { + return func(t *structTransformer) { + t.typeTransformer = transformer + } +} + +// WithResolverTransformer overrides how column resolver will be determined. +// DefaultResolverTransformer is used as the default. +func WithResolverTransformer(transformer ResolverTransformer) StructTransformerOption { + return func(t *structTransformer) { + t.resolverTransformer = transformer + } +} + +// WithIgnoreInTestsTransformer overrides how column ignoreInTests will be determined. +// DefaultIgnoreInTestsTransformer is used as the default. +func WithIgnoreInTestsTransformer(transformer IgnoreInTestsTransformer) StructTransformerOption { + return func(t *structTransformer) { + t.ignoreInTestsTransformer = transformer + } +} + +// WithPrimaryKeys allows to specify what struct fields should be used as primary keys +func WithPrimaryKeys(fields ...string) StructTransformerOption { + return func(t *structTransformer) { + t.pkFields = fields + } +} + +// WithPrimaryKeyComponents allows to specify what struct fields should be used as primary key components +func WithPrimaryKeyComponents(fields ...string) StructTransformerOption { + return func(t *structTransformer) { + t.pkComponentFields = fields + } +} diff --git a/plugin-sdk/transformers/resolver.go b/plugin-sdk/transformers/resolver.go new file mode 100644 index 00000000..4a996e95 --- /dev/null +++ b/plugin-sdk/transformers/resolver.go @@ -0,0 +1,15 @@ +package transformers + +import ( + "reflect" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +type ResolverTransformer func(field reflect.StructField, path string) schema.ColumnResolver + +func DefaultResolverTransformer(_ reflect.StructField, path string) schema.ColumnResolver { + return schema.PathResolver(path) +} + +var _ ResolverTransformer = DefaultResolverTransformer diff --git a/plugin-sdk/transformers/struct.go b/plugin-sdk/transformers/struct.go new file mode 100644 index 00000000..906f2927 --- /dev/null +++ b/plugin-sdk/transformers/struct.go @@ -0,0 +1,236 @@ +package transformers + +import ( + "fmt" + "reflect" + "slices" + + "github.com/thoas/go-funk" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +type structTransformer struct { + table *schema.Table + skipFields []string + nameTransformer NameTransformer + typeTransformer TypeTransformer + resolverTransformer ResolverTransformer + ignoreInTestsTransformer IgnoreInTestsTransformer + unwrapAllEmbeddedStructFields bool + structFieldsToUnwrap []string + pkFields []string + pkFieldsFound []string + pkComponentFields []string + pkComponentFieldsFound []string +} + +func isFieldStruct(reflectType reflect.Type) bool { + switch reflectType.Kind() { + case reflect.Struct: + return true + case reflect.Ptr: + return reflectType.Elem().Kind() == reflect.Struct + default: + return false + } +} + +func isTypeIgnored(t reflect.Type) bool { + switch t.Kind() { + case reflect.Func, + reflect.Chan, + reflect.UnsafePointer: + return true + default: + return false + } +} + +func (t *structTransformer) getUnwrappedFields(field reflect.StructField) []reflect.StructField { + reflectType := field.Type + if reflectType.Kind() == reflect.Ptr { + reflectType = reflectType.Elem() + } + + fields := make([]reflect.StructField, 0) + for i := 0; i < reflectType.NumField(); i++ { + sf := reflectType.Field(i) + if t.ignoreField(sf) { + continue + } + + fields = append(fields, sf) + } + return fields +} + +func (t *structTransformer) unwrapField(field reflect.StructField) error { + unwrappedFields := t.getUnwrappedFields(field) + var parent *reflect.StructField + // For non embedded structs we need to add the parent field name to the path + if !field.Anonymous { + parent = &field + } + for _, f := range unwrappedFields { + if err := t.addColumnFromField(f, parent); err != nil { + return fmt.Errorf("failed to add column from field %s: %w", f.Name, err) + } + } + return nil +} + +func (t *structTransformer) shouldUnwrapField(field reflect.StructField) bool { + switch { + case !isFieldStruct(field.Type): + return false + case slices.Contains(t.structFieldsToUnwrap, field.Name): + return true + case !field.Anonymous: + return false + case t.unwrapAllEmbeddedStructFields: + return true + default: + return false + } +} + +func (t *structTransformer) ignoreField(field reflect.StructField) bool { + switch { + case len(field.Name) == 0, + slices.Contains(t.skipFields, field.Name), + !field.IsExported(), + isTypeIgnored(field.Type): + return true + default: + return false + } +} + +func (t *structTransformer) addColumnFromField(field reflect.StructField, parent *reflect.StructField) error { + if t.ignoreField(field) { + return nil + } + + columnType, err := t.typeTransformer(field) + if err != nil { + return fmt.Errorf("failed to transform type for field %s: %w", field.Name, err) + } + + if columnType == nil { + columnType, err = DefaultTypeTransformer(field) + if err != nil { + return fmt.Errorf("failed to transform type for field %s: %w", field.Name, err) + } + } + if columnType == nil { + return nil // ignored + } + + path := field.Name + name, err := t.nameTransformer(field) + if err != nil { + return fmt.Errorf("failed to transform field name for field %s: %w", field.Name, err) + } + // skip field if there is no name + if name == "" { + return nil + } + if parent != nil { + parentName, err := t.nameTransformer(*parent) + if err != nil { + return fmt.Errorf("failed to transform field name for parent field %s: %w", parent.Name, err) + } + name = parentName + "_" + name + path = parent.Name + `.` + path + } + if t.table.Columns.Get(name) != nil { + return nil + } + + resolver := t.resolverTransformer(field, path) + if resolver == nil { + resolver = DefaultResolverTransformer(field, path) + } + + column := schema.Column{ + Name: name, + Type: columnType, + Resolver: resolver, + IgnoreInTests: t.ignoreInTestsTransformer(field), + } + + for _, pk := range t.pkFields { + if pk == path { + // use path to allow the following + // 1. Don't duplicate the PK fields if the unwrapped struct contains a fields with the same name + // 2. Allow specifying the nested unwrapped field as part of the PK. + column.PrimaryKey = true + t.pkFieldsFound = append(t.pkFieldsFound, pk) + } + } + + for _, pk := range t.pkComponentFields { + if pk == path { + // use path to allow the following + // 1. Don't duplicate the PK fields if the unwrapped struct contains a fields with the same name + // 2. Allow specifying the nested unwrapped field as part of the PK. + column.PrimaryKeyComponent = true + t.pkComponentFieldsFound = append(t.pkComponentFieldsFound, pk) + } + } + + t.table.Columns = append(t.table.Columns, column) + + return nil +} + +func TransformWithStruct(st any, opts ...StructTransformerOption) schema.Transform { + t := &structTransformer{ + nameTransformer: DefaultNameTransformer, + typeTransformer: DefaultTypeTransformer, + resolverTransformer: DefaultResolverTransformer, + ignoreInTestsTransformer: DefaultIgnoreInTestsTransformer, + } + for _, opt := range opts { + opt(t) + } + + return func(table *schema.Table) error { + t.table = table + e := reflect.ValueOf(st) + if e.Kind() == reflect.Pointer { + e = e.Elem() + } + if e.Kind() == reflect.Slice { + e = reflect.MakeSlice(e.Type(), 1, 1).Index(0) + } + if e.Kind() != reflect.Struct { + return fmt.Errorf("expected struct, got %s", e.Kind()) + } + eType := e.Type() + for i := 0; i < e.NumField(); i++ { + field := eType.Field(i) + + switch { + case t.shouldUnwrapField(field): + if err := t.unwrapField(field); err != nil { + return err + } + default: + if err := t.addColumnFromField(field, nil); err != nil { + return fmt.Errorf("failed to add column for field %s: %w", field.Name, err) + } + } + } + // Validate that all expected PK fields were found + if diff := funk.SubtractString(t.pkFields, t.pkFieldsFound); len(diff) > 0 { + return fmt.Errorf("failed to create all of the desired primary keys: %v", diff) + } + + if diff := funk.SubtractString(t.pkComponentFields, t.pkComponentFieldsFound); len(diff) > 0 { + return fmt.Errorf("failed to find all of the desired primary key components: %v", diff) + } + return nil + } +} diff --git a/plugin-sdk/transformers/struct_test.go b/plugin-sdk/transformers/struct_test.go new file mode 100644 index 00000000..2812cf01 --- /dev/null +++ b/plugin-sdk/transformers/struct_test.go @@ -0,0 +1,483 @@ +package transformers + +import ( + "net" + "reflect" + "slices" + "testing" + "time" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/google/go-cmp/cmp" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" +) + +type ( + embeddedStruct struct { + EmbeddedString string + IntCol int `json:"int_col,omitempty"` + } + + testStruct struct { + // IntCol this is an example documentation comment + IntCol int `json:"int_col,omitempty"` + Int64Col int64 `json:"int64_col,omitempty"` + StringCol string `json:"string_col,omitempty"` + FloatCol float64 `json:"float_col,omitempty"` + BoolCol bool `json:"bool_col,omitempty"` + JSONCol struct { + IntCol int `json:"int_col,omitempty"` + StringCol string `json:"string_col,omitempty"` + } + IntArrayCol []int `json:"int_array_col,omitempty"` + IntPointerArrayCol []*int `json:"int_pointer_array_col,omitempty"` + + StringArrayCol []string `json:"string_array_col,omitempty"` + StringPointerArrayCol []*string `json:"string_pointer_array_col,omitempty"` + + InetCol net.IP `json:"inet_col,omitempty"` + InetPointerCol *net.IP `json:"inet_pointer_col,omitempty"` + + ByteArrayCol []byte `json:"byte_array_col,omitempty"` + + AnyArrayCol []any `json:"any_array_col,omitempty"` + + TimeCol time.Time `json:"time_col,omitempty"` + TimePointerCol *time.Time `json:"time_pointer_col,omitempty"` + JSONTag *string `json:"json_tag"` + SkipJSONTag *string `json:"-"` + NoJSONTag *string + *embeddedStruct + } + testStructWithEmbeddedStruct struct { + IntCol int `json:"int_col,omitempty"` + *testStruct + *embeddedStruct + } + testStructWithNonEmbeddedStruct struct { + IntCol int `json:"int_col,omitempty"` + TestStruct *testStruct + NonEmbedded *embeddedStruct + } + + testSliceStruct []struct { + IntCol int + } + + testPKStruct struct { + Parent string `json:"parent"` + Name string `json:"name"` + Version int `json:"version"` + } + + testFunnyStruct struct { + AFunnyLookingField string `json:"OS-EXT:a-funny-looking-field"` + AFieldWithCamelCaseName string `json:"camelCaseName"` + } + + testStructWithAny struct { + IntCol int `json:"int_col"` + Properties any + } +) + +var ( + expectedColumns = []schema.Column{ + { + Name: "int_col", + Type: arrow.PrimitiveTypes.Int64, + }, + { + Name: "int64_col", + Type: arrow.PrimitiveTypes.Int64, + }, + { + Name: "string_col", + Type: arrow.BinaryTypes.String, + }, + { + Name: "float_col", + Type: arrow.PrimitiveTypes.Float64, + }, + { + Name: "bool_col", + Type: arrow.FixedWidthTypes.Boolean, + }, + { + Name: "json_col", + Type: types.ExtensionTypes.JSON, + }, + { + Name: "int_array_col", + Type: arrow.ListOf(arrow.PrimitiveTypes.Int64), + }, + { + Name: "int_pointer_array_col", + Type: arrow.ListOf(arrow.PrimitiveTypes.Int64), + }, + { + Name: "string_array_col", + Type: arrow.ListOf(arrow.BinaryTypes.String), + }, + { + Name: "string_pointer_array_col", + Type: arrow.ListOf(arrow.BinaryTypes.String), + }, + { + Name: "inet_col", + Type: types.ExtensionTypes.Inet, + }, + { + Name: "inet_pointer_col", + Type: types.ExtensionTypes.Inet, + }, + { + Name: "byte_array_col", + Type: arrow.BinaryTypes.Binary, + }, + { + Name: "any_array_col", + Type: types.ExtensionTypes.JSON, + }, + { + Name: "time_col", + Type: arrow.FixedWidthTypes.Timestamp_us, + }, + { + Name: "time_pointer_col", + Type: arrow.FixedWidthTypes.Timestamp_us, + }, + { + Name: "json_tag", + Type: arrow.BinaryTypes.String, + }, + { + Name: "no_json_tag", + Type: arrow.BinaryTypes.String, + }, + } + expectedTestTable = schema.Table{ + Name: "test_struct", + Columns: expectedColumns, + } + expectedTestTableEmbeddedStruct = schema.Table{ + Name: "test_struct", + Columns: append(expectedColumns, schema.Column{Name: "embedded_string", Type: arrow.BinaryTypes.String}), + } + expectedTestTableEmbeddedStructWithTopLevelPK = schema.Table{ + Name: "test_struct", + Columns: func(base schema.ColumnList) schema.ColumnList { + cols := slices.Clone(base) + cols = append(cols, schema.Column{Name: "embedded_string", Type: arrow.BinaryTypes.String}) + cols[cols.Index("int_col")].PrimaryKey = true + return cols + }(expectedColumns), + } + expectedTestTableEmbeddedStructWithUnwrappedPK = schema.Table{ + Name: "test_struct", + Columns: append( + expectedColumns, schema.Column{ + Name: "embedded_string", + Type: arrow.BinaryTypes.String, + PrimaryKey: true, + }), + } + expectedTestTableNonEmbeddedStruct = schema.Table{ + Name: "test_struct", + Columns: schema.ColumnList{ + schema.Column{Name: "int_col", Type: arrow.PrimitiveTypes.Int64}, + // Should not be unwrapped + schema.Column{Name: "test_struct", Type: types.ExtensionTypes.JSON}, + // Should be unwrapped + schema.Column{Name: "non_embedded_embedded_string", Type: arrow.BinaryTypes.String}, + schema.Column{Name: "non_embedded_int_col", Type: arrow.PrimitiveTypes.Int64}, + }, + } + expectedTestTableNonEmbeddedStructWithTopLevelPK = schema.Table{ + Name: "test_struct", + Columns: schema.ColumnList{ + schema.Column{ + Name: "int_col", + Type: arrow.PrimitiveTypes.Int64, + PrimaryKey: true, + }, + // Should not be unwrapped + schema.Column{Name: "test_struct", Type: types.ExtensionTypes.JSON}, + // Should be unwrapped + schema.Column{ + Name: "non_embedded_embedded_string", + Type: arrow.BinaryTypes.String, + }, + schema.Column{Name: "non_embedded_int_col", Type: arrow.PrimitiveTypes.Int64}, + }, + } + expectedTestTableNonEmbeddedStructWithUnwrappedPK = schema.Table{ + Name: "test_struct", + Columns: schema.ColumnList{ + // shouldn't be PK + schema.Column{Name: "int_col", Type: arrow.PrimitiveTypes.Int64}, + // Should not be unwrapped + schema.Column{Name: "test_struct", Type: types.ExtensionTypes.JSON}, + // Should be unwrapped + schema.Column{ + Name: "non_embedded_embedded_string", + Type: arrow.BinaryTypes.String, + }, + // should be PK + schema.Column{ + Name: "non_embedded_int_col", + Type: arrow.PrimitiveTypes.Int64, + PrimaryKey: true, + }, + }, + } + expectedTestSliceStruct = schema.Table{ + Name: "test_struct", + Columns: schema.ColumnList{ + { + Name: "int_col", + Type: arrow.PrimitiveTypes.Int64, + }, + }, + } + + expectedTableWithPKs = schema.Table{ + Name: "test_pk_struct", + Columns: schema.ColumnList{ + { + Name: "parent", + Type: arrow.BinaryTypes.String, + PrimaryKey: true, + }, + { + Name: "name", + Type: arrow.BinaryTypes.String, + PrimaryKey: true, + }, + { + Name: "version", + Type: arrow.PrimitiveTypes.Int64, + }, + }, + } + + expectedFunnyTable = schema.Table{ + Name: "test_funny_struct", + Columns: schema.ColumnList{ + { + Name: "a_funny_looking_field", + Type: arrow.BinaryTypes.String, + }, + { + Name: "camel_case_name", + Type: arrow.BinaryTypes.String, + }, + }, + } + + expectedTestTableStructWithAny = schema.Table{ + Name: "test_embedded_struct_with_any", + Columns: schema.ColumnList{ + { + Name: "int_col", + Type: arrow.PrimitiveTypes.Int64, + }, + }, + } + + expectedTestTableStructWithCustomAny = schema.Table{ + Name: "test_embedded_struct_with_custom_any", + Columns: schema.ColumnList{ + { + Name: "int_col", + Type: arrow.PrimitiveTypes.Int64, + }, + { + Name: "properties", + Type: types.ExtensionTypes.JSON, + }, + }, + } +) + +func TestTableFromGoStruct(t *testing.T) { + type args struct { + testStruct any + options []StructTransformerOption + } + + tests := []struct { + name string + args args + want schema.Table + wantErr bool + }{ + { + name: "should generate table from struct with default options", + args: args{ + testStruct: testStruct{}, + }, + want: expectedTestTable, + }, + { + name: "should unwrap all embedded structs when option is set", + args: args{ + testStruct: testStructWithEmbeddedStruct{}, + options: []StructTransformerOption{ + WithUnwrapAllEmbeddedStructs(), + }, + }, + want: expectedTestTableEmbeddedStruct, + }, + { + name: "should unwrap all embedded structs when option is set and use top-level field as PK", + args: args{ + testStruct: testStructWithEmbeddedStruct{}, + options: []StructTransformerOption{ + WithUnwrapAllEmbeddedStructs(), + WithPrimaryKeys("IntCol"), + }, + }, + want: expectedTestTableEmbeddedStructWithTopLevelPK, + }, + { + name: "should unwrap all embedded structs when option is set and use its field as PK", + args: args{ + testStruct: testStructWithEmbeddedStruct{}, + options: []StructTransformerOption{ + WithUnwrapAllEmbeddedStructs(), + WithPrimaryKeys("EmbeddedString"), + }, + }, + want: expectedTestTableEmbeddedStructWithUnwrappedPK, + }, + { + name: "should unwrap specific structs when option is set", + args: args{ + testStruct: testStructWithNonEmbeddedStruct{}, + options: []StructTransformerOption{ + WithUnwrapStructFields("NonEmbedded"), + }, + }, + want: expectedTestTableNonEmbeddedStruct, + }, + { + name: "should unwrap specific structs when option is set and use top level field as PK", + args: args{ + testStruct: testStructWithNonEmbeddedStruct{}, + options: []StructTransformerOption{ + WithUnwrapStructFields("NonEmbedded"), + WithPrimaryKeys("IntCol"), + }, + }, + want: expectedTestTableNonEmbeddedStructWithTopLevelPK, + }, + { + name: "should unwrap specific structs when option is set and use its field as PK", + args: args{ + testStruct: testStructWithNonEmbeddedStruct{}, + options: []StructTransformerOption{ + WithUnwrapStructFields("NonEmbedded"), + WithPrimaryKeys("NonEmbedded.IntCol"), + }, + }, + want: expectedTestTableNonEmbeddedStructWithUnwrappedPK, + }, + { + name: "should generate table from slice struct", + args: args{ + testStruct: testSliceStruct{}, + }, + want: expectedTestSliceStruct, + }, + { + name: "Should configure primary keys when options are set", + args: args{ + testStruct: testPKStruct{}, + options: []StructTransformerOption{ + WithPrimaryKeys("Parent", "Name"), + }, + }, + want: expectedTableWithPKs, + }, + { + name: "Should return an error when a PK Field is not found", + args: args{ + testStruct: testPKStruct{}, + options: []StructTransformerOption{ + WithPrimaryKeys("Parent", "Name", "InvalidColumns"), + }, + }, + want: expectedTableWithPKs, + wantErr: true, + }, + { + name: "Should properly transform structs with funny looking fields", + args: args{ + testStruct: testFunnyStruct{}, + }, + want: expectedFunnyTable, + }, + { + name: "Ignore any-type fields by default", + args: args{ + testStruct: testStructWithAny{}, + }, + want: expectedTestTableStructWithAny, + }, + { + name: "Should be able to override any-type fields with a type", + args: args{ + testStruct: testStructWithAny{}, + options: []StructTransformerOption{ + WithTypeTransformer(func(f reflect.StructField) (arrow.DataType, error) { + if f.Type.Kind() == reflect.Interface { + return types.ExtensionTypes.JSON, nil + } + return nil, nil + }), + }, + }, + want: expectedTestTableStructWithCustomAny, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + table := schema.Table{ + Name: "test", + Columns: schema.ColumnList{}, + } + transformer := TransformWithStruct(tt.args.testStruct, tt.args.options...) + err := transformer(&table) + if err != nil { + if tt.wantErr { + return + } + t.Fatal(err) + } + if tt.wantErr { + t.Fatal("expected error, got none") + } + + for i, col := range table.Columns { + if !arrow.TypeEqual(col.Type, tt.want.Columns[i].Type) { + t.Fatalf("column %q does not match expected type. got %v, want %v", col.Name, col.Type, tt.want.Columns[i].Type) + } + } + for _, exc := range tt.want.Columns { + if c := table.Column(exc.Name); c == nil { + t.Fatalf("column %q not found. want: %v", exc.Name, exc.Type) + } + } + + if diff := cmp.Diff(table.PrimaryKeys(), tt.want.PrimaryKeys()); diff != "" { + t.Fatalf("table does not match expected. diff (-got, +want): %v", diff) + } + }) + } +} diff --git a/plugin-sdk/transformers/tables.go b/plugin-sdk/transformers/tables.go new file mode 100644 index 00000000..3d1bb9df --- /dev/null +++ b/plugin-sdk/transformers/tables.go @@ -0,0 +1,47 @@ +package transformers + +import ( + "fmt" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +// Set parent links on relational tables +func SetParents(tables schema.Tables, parent *schema.Table) { + for _, table := range tables { + table.Parent = parent + SetParents(table.Relations, table) + } +} + +// TransformTables runs given Tables' transformers as defined in the table definitions, and recursively on the relations. +// To apply additional transformers, use Apply. +func TransformTables(tables schema.Tables) error { + for _, table := range tables { + if table.Transform != nil { + if err := table.Transform(table); err != nil { + return fmt.Errorf("failed to transform table %s: %w", table.Name, err) + } + } + if err := TransformTables(table.Relations); err != nil { + return err + } + } + return nil +} + +// Apply applies the given transformers to the given Tables, and recursively on the relations. +// This is useful for applying transformers that are not defined in the table definitions. To apply the table-definition transformers, use TransformTables. +func Apply(tables schema.Tables, extraTransformers ...schema.Transform) error { + for _, table := range tables { + for _, tf := range extraTransformers { + if err := tf(table); err != nil { + return fmt.Errorf("failed to transform table %s: %w", table.Name, err) + } + } + if err := Apply(table.Relations, extraTransformers...); err != nil { + return err + } + } + return nil +} diff --git a/plugin-sdk/transformers/type.go b/plugin-sdk/transformers/type.go new file mode 100644 index 00000000..5fcaf729 --- /dev/null +++ b/plugin-sdk/transformers/type.go @@ -0,0 +1,72 @@ +package transformers + +import ( + "fmt" + "net" + "reflect" + "time" + + "github.com/apache/arrow/go/v16/arrow" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" +) + +type TypeTransformer func(reflect.StructField) (arrow.DataType, error) + +func defaultGoTypeToSchemaType(v reflect.Type) (arrow.DataType, error) { + // Non-primitive types + if v == reflect.TypeOf(net.IP{}) { + return types.ExtensionTypes.Inet, nil + } + + k := v.Kind() + switch k { + case reflect.Pointer: + return defaultGoTypeToSchemaType(v.Elem()) + case reflect.String: + return arrow.BinaryTypes.String, nil + case reflect.Bool: + return arrow.FixedWidthTypes.Boolean, nil + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return arrow.PrimitiveTypes.Int64, nil + case reflect.Float32, reflect.Float64: + return arrow.PrimitiveTypes.Float64, nil + case reflect.Map: + return types.ExtensionTypes.JSON, nil + case reflect.Struct: + if v == reflect.TypeOf(time.Time{}) { + return arrow.FixedWidthTypes.Timestamp_us, nil + } + return types.ExtensionTypes.JSON, nil + case reflect.Slice: + switch v.Elem().Kind() { + case reflect.Uint8: + return arrow.BinaryTypes.Binary, nil + case reflect.Interface: + return types.ExtensionTypes.JSON, nil + default: + elemValueType, err := defaultGoTypeToSchemaType(v.Elem()) + if err != nil { + return nil, err + } + // if it's already JSON then we don't want to create list of JSON + if arrow.TypeEqual(elemValueType, types.ExtensionTypes.JSON) { + return elemValueType, nil + } + return arrow.ListOf(elemValueType), nil + } + + case reflect.Interface: + return nil, nil // silently ignore + + default: + return nil, fmt.Errorf("unsupported type: %s", k) + } +} + +func DefaultTypeTransformer(v reflect.StructField) (arrow.DataType, error) { + return defaultGoTypeToSchemaType(v.Type) +} + +var _ TypeTransformer = DefaultTypeTransformer diff --git a/plugin-sdk/types/extensions.go b/plugin-sdk/types/extensions.go new file mode 100644 index 00000000..9a37622e --- /dev/null +++ b/plugin-sdk/types/extensions.go @@ -0,0 +1,15 @@ +package types + +import "github.com/apache/arrow/go/v16/arrow" + +var ExtensionTypes = struct { + UUID arrow.ExtensionType + Inet arrow.ExtensionType + MAC arrow.ExtensionType + JSON arrow.ExtensionType +}{ + UUID: NewUUIDType(), + Inet: NewInetType(), + MAC: NewMACType(), + JSON: NewJSONType(), +} diff --git a/plugin-sdk/types/extensions_test.go b/plugin-sdk/types/extensions_test.go new file mode 100644 index 00000000..36fcd5f9 --- /dev/null +++ b/plugin-sdk/types/extensions_test.go @@ -0,0 +1,145 @@ +package types + +import ( + "testing" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValueStrRoundTrip(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + + cases := []struct { + arr arrow.Array + builder array.Builder + }{ + { + arr: func() arrow.Array { + b := NewInetBuilder(array.NewExtensionBuilder(mem, NewInetType())) + defer b.Release() + + b.AppendNull() + b.Append(mustParseInet("192.168.0.0/24")) + b.AppendNull() + b.Append(mustParseInet("192.168.0.0/25")) + b.AppendNull() + + return b.NewInetArray() + }(), + builder: NewInetBuilder(array.NewExtensionBuilder(mem, NewInetType())), + }, + { + arr: func() arrow.Array { + b := NewJSONBuilder(array.NewExtensionBuilder(mem, NewJSONType())) + defer b.Release() + + b.AppendNull() + b.Append(map[string]any{"a": 1, "b": 2}) + b.AppendNull() + b.Append([]any{1, 2, 3}) + b.AppendNull() + b.Append(map[string]any{"MyKey": "A\u0026B"}) + b.AppendNull() + + return b.NewJSONArray() + }(), + builder: NewJSONBuilder(array.NewExtensionBuilder(mem, NewJSONType())), + }, + { + arr: func() arrow.Array { + b := NewMACBuilder(array.NewExtensionBuilder(mem, NewMACType())) + defer b.Release() + + b.AppendNull() + b.Append(mustParseMAC("00:00:00:00:00:01")) + b.AppendNull() + b.Append(mustParseMAC("00:00:00:00:00:02")) + b.AppendNull() + + return b.NewMACArray() + }(), + builder: NewMACBuilder(array.NewExtensionBuilder(mem, NewMACType())), + }, + { + arr: func() arrow.Array { + b := NewUUIDBuilder(array.NewExtensionBuilder(mem, NewUUIDType())) + defer b.Release() + + b.AppendNull() + b.Append(uuid.NameSpaceURL) + b.AppendNull() + b.Append(uuid.NameSpaceDNS) + b.AppendNull() + + return b.NewUUIDArray() + }(), + builder: NewUUIDBuilder(array.NewExtensionBuilder(mem, NewUUIDType())), + }, + } + + for _, tc := range cases { + t.Run(tc.arr.DataType().(arrow.ExtensionType).ExtensionName(), func(t *testing.T) { + defer tc.arr.Release() + defer tc.builder.Release() + t.Helper() + + for i := 0; i < tc.arr.Len(); i++ { + assert.NoError(t, tc.builder.AppendValueFromString(tc.arr.ValueStr(i))) + } + + arr := tc.builder.NewArray() + defer arr.Release() + + require.True(t, array.Equal(tc.arr, arr)) + }) + } +} + +func TestStorageArrayConv(t *testing.T) { + const amount = 100 + cases := []arrow.ExtensionType{ExtensionTypes.UUID, ExtensionTypes.MAC, ExtensionTypes.JSON, ExtensionTypes.Inet} + for _, dt := range cases { + t.Run(dt.String(), func(t *testing.T) { + storageBuilder := array.NewBuilder(memory.DefaultAllocator, dt.StorageType()) + defer storageBuilder.Release() + builder := array.NewBuilder(memory.DefaultAllocator, dt) + defer builder.Release() + + for i := 0; i < amount; i++ { + if i%2 == 0 { + storageBuilder.AppendNull() + builder.AppendNull() + continue + } + storageBuilder.AppendEmptyValue() + builder.AppendEmptyValue() + } + + storage := storageBuilder.NewArray() + defer storage.Release() + arr := builder.NewArray().(array.ExtensionArray) + defer arr.Release() + + // check matching + assert.True(t, array.Equal(storage, arr.Storage())) + + // check that creating extension from storage matches + fromStorage := array.NewExtensionArrayWithStorage(dt, storage) + defer fromStorage.Release() + + assert.True(t, array.Equal(arr, fromStorage)) + + // assert that no issues are in the fromStorage array + for i := 0; i < fromStorage.Len(); i++ { + assert.NotPanics(t, func() { fromStorage.ValueStr(i) }) + assert.Equal(t, arr.ValueStr(i), fromStorage.ValueStr(i)) + } + }) + } +} diff --git a/plugin-sdk/types/inet.go b/plugin-sdk/types/inet.go new file mode 100644 index 00000000..b3c2df30 --- /dev/null +++ b/plugin-sdk/types/inet.go @@ -0,0 +1,249 @@ +package types + +import ( + "bytes" + "fmt" + "net" + "reflect" + "strings" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/goccy/go-json" +) + +type InetBuilder struct { + *array.ExtensionBuilder +} + +func NewInetBuilder(builder *array.ExtensionBuilder) *InetBuilder { + return &InetBuilder{ExtensionBuilder: builder} +} + +func (b *InetBuilder) Append(v *net.IPNet) { + if v == nil { + b.AppendNull() + return + } + b.ExtensionBuilder.Builder.(*array.BinaryBuilder).Append([]byte(v.String())) +} + +func (b *InetBuilder) UnsafeAppend(v *net.IPNet) { + b.ExtensionBuilder.Builder.(*array.BinaryBuilder).UnsafeAppend([]byte(v.String())) +} + +func (b *InetBuilder) AppendValues(v []*net.IPNet, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + data := make([][]byte, len(v)) + for i, v := range v { + if len(valid) > 0 && !valid[i] { + continue + } + data[i] = []byte(v.String()) + } + b.ExtensionBuilder.Builder.(*array.BinaryBuilder).AppendValues(data, valid) +} + +func (b *InetBuilder) AppendValueFromString(s string) error { + if s == array.NullValueStr { + b.AppendNull() + return nil + } + _, data, err := net.ParseCIDR(s) + if err != nil { + return err + } + b.Append(data) + return nil +} + +func (b *InetBuilder) UnmarshalOne(dec *json.Decoder) error { + t, err := dec.Token() + if err != nil { + return err + } + + var val *net.IPNet + var ip net.IP + switch v := t.(type) { + case string: + ip, val, err = net.ParseCIDR(v) + if err != nil { + return err + } + val.IP = ip + case []byte: + ip, val, err = net.ParseCIDR(string(v)) + if err != nil { + return err + } + val.IP = ip + case nil: + b.AppendNull() + return nil + default: + return &json.UnmarshalTypeError{ + Value: fmt.Sprint(t), + Type: reflect.TypeOf([]byte{}), + Offset: dec.InputOffset(), + Struct: "String", + } + } + + b.Append(val) + return nil +} + +func (b *InetBuilder) Unmarshal(dec *json.Decoder) error { + for dec.More() { + if err := b.UnmarshalOne(dec); err != nil { + return err + } + } + return nil +} + +func (b *InetBuilder) UnmarshalJSON(data []byte) error { + dec := json.NewDecoder(bytes.NewReader(data)) + t, err := dec.Token() + if err != nil { + return err + } + + if delim, ok := t.(json.Delim); !ok || delim != '[' { + return fmt.Errorf("fixed size binary builder must unpack from json array, found %s", delim) + } + + return b.Unmarshal(dec) +} + +func (b *InetBuilder) NewInetArray() *InetArray { + return b.NewExtensionArray().(*InetArray) +} + +// InetArray is a simple array which is a FixedSizeBinary(16) +type InetArray struct { + array.ExtensionArrayBase +} + +func (a *InetArray) String() string { + arr := a.Storage().(*array.Binary) + o := new(strings.Builder) + o.WriteString("[") + for i := 0; i < arr.Len(); i++ { + if i > 0 { + o.WriteString(" ") + } + switch { + case a.IsNull(i): + o.WriteString(array.NullValueStr) + default: + fmt.Fprintf(o, "%q", a.ValueStr(i)) + } + } + o.WriteString("]") + return o.String() +} + +func (a *InetArray) Value(i int) *net.IPNet { + if a.IsNull(i) { + return nil + } + cidr := string(a.Storage().(*array.Binary).Value(i)) + if len(cidr) == 0 { + return &net.IPNet{ + IP: net.IPv4zero, + Mask: make(net.IPMask, len(net.IPv4zero)), + } + } + ip, ipnet, err := net.ParseCIDR(cidr) + if err != nil { + panic(fmt.Errorf("invalid ip+net: %w", err)) + } + ipnet.IP = ip + + return ipnet +} + +func (a *InetArray) ValueStr(i int) string { + switch { + case a.IsNull(i): + return array.NullValueStr + default: + return a.Value(i).String() + } +} + +func (a *InetArray) GetOneForMarshal(i int) any { + if a.IsNull(i) { + return nil + } + return string(a.Storage().(*array.Binary).Value(i)) +} + +func (a *InetArray) MarshalJSON() ([]byte, error) { + arr := a.Storage().(*array.Binary) + values := make([]any, a.Len()) + for i := 0; i < a.Len(); i++ { + if a.IsValid(i) { + values[i] = string(arr.Value(i)) + } else { + values[i] = nil + } + } + return json.Marshal(values) +} + +// InetType is a simple extension type that represents a BinaryType +// to be used for representing IP Addresses and CIDRs +type InetType struct { + arrow.ExtensionBase +} + +// NewInetType is a convenience function to create an instance of InetType +// with the correct storage type +func NewInetType() *InetType { + return &InetType{ExtensionBase: arrow.ExtensionBase{Storage: &arrow.BinaryType{}}} +} + +// ArrayType returns TypeOf(InetArray{}) for constructing Inet arrays +func (*InetType) ArrayType() reflect.Type { + return reflect.TypeOf(InetArray{}) +} + +func (*InetType) ExtensionName() string { + return "inet" +} + +func (*InetType) String() string { + return "inet" +} + +// Serialize returns "inet-serialized" for testing proper metadata passing +func (*InetType) Serialize() string { + return "inet-serialized" +} + +// Deserialize expects storageType to be BinaryType and the data to be +// "inet-serialized" in order to correctly create a InetType for testing deserialize. +func (*InetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) { + if data != "inet-serialized" { + return nil, fmt.Errorf("type identifier did not match: '%s'", data) + } + if !arrow.TypeEqual(storageType, &arrow.BinaryType{}) { + return nil, fmt.Errorf("invalid storage type for InetType: %s", storageType.Name()) + } + return NewInetType(), nil +} + +// ExtensionEquals returns true if both extensions have the same name +func (u *InetType) ExtensionEquals(other arrow.ExtensionType) bool { + return u.ExtensionName() == other.ExtensionName() +} + +func (*InetType) NewBuilder(bldr *array.ExtensionBuilder) array.Builder { + return NewInetBuilder(bldr) +} diff --git a/plugin-sdk/types/inet_test.go b/plugin-sdk/types/inet_test.go new file mode 100644 index 00000000..6d675220 --- /dev/null +++ b/plugin-sdk/types/inet_test.go @@ -0,0 +1,66 @@ +package types + +import ( + "net" + "testing" + + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/stretchr/testify/require" +) + +func mustParseInet(s string) *net.IPNet { + ip, ipnet, err := net.ParseCIDR(s) + if err != nil { + panic(err) + } + ipnet.IP = ip + return ipnet +} + +func TestInetBuilder(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + + b := NewInetBuilder(array.NewExtensionBuilder(mem, NewInetType())) + + b.Append(mustParseInet("192.168.0.0/24")) + b.AppendNull() + b.Append(mustParseInet("192.168.0.0/25")) + b.AppendNull() + + require.Equal(t, 4, b.Len(), "unexpected Len()") + require.Equal(t, 2, b.NullN(), "unexpected NullN()") + + values := []*net.IPNet{ + mustParseInet("192.168.0.0/26"), + mustParseInet("192.168.0.0/27"), + } + b.AppendValues(values, nil) + b.Append(mustParseInet("192.168.0.1/24")) + + require.Equal(t, 7, b.Len(), "unexpected Len()") + + a := b.NewArray() + + // check state of builder after NewInetBuilder + require.Zero(t, b.Len(), "unexpected ArrayBuilder.Len(), did not reset state") + require.Zero(t, b.Cap(), "unexpected ArrayBuilder.Cap(), did not reset state") + require.Zero(t, b.NullN(), "unexpected ArrayBuilder.NullN(), did not reset state") + + require.Equal(t, `["192.168.0.0/24" (null) "192.168.0.0/25" (null) "192.168.0.0/26" "192.168.0.0/27" "192.168.0.1/24"]`, a.String()) + st, err := a.MarshalJSON() + require.NoError(t, err) + + b.Release() + a.Release() + + b = NewInetBuilder(array.NewExtensionBuilder(mem, NewInetType())) + err = b.UnmarshalJSON(st) + require.NoError(t, err) + + a = b.NewArray() + require.Equal(t, `["192.168.0.0/24" (null) "192.168.0.0/25" (null) "192.168.0.0/26" "192.168.0.0/27" "192.168.0.1/24"]`, a.String()) + b.Release() + a.Release() +} diff --git a/plugin-sdk/types/json.go b/plugin-sdk/types/json.go new file mode 100644 index 00000000..f921f67f --- /dev/null +++ b/plugin-sdk/types/json.go @@ -0,0 +1,243 @@ +package types + +import ( + "bytes" + "fmt" + "reflect" + "strings" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/goccy/go-json" +) + +type JSONBuilder struct { + *array.ExtensionBuilder +} + +func NewJSONBuilder(builder *array.ExtensionBuilder) *JSONBuilder { + return &JSONBuilder{ExtensionBuilder: builder} +} + +func (b *JSONBuilder) AppendBytes(v []byte) { + if v == nil { + b.AppendNull() + return + } + + b.ExtensionBuilder.Builder.(*array.BinaryBuilder).Append(v) +} + +func (b *JSONBuilder) Append(v any) { + if v == nil { + b.AppendNull() + return + } + + // per https://github.com/cloudquery/plugin-sdk/issues/622 + data, err := json.MarshalWithOption(v, json.DisableHTMLEscape()) + if err != nil { + panic(err) + } + + b.ExtensionBuilder.Builder.(*array.BinaryBuilder).Append(data) +} + +func (b *JSONBuilder) UnsafeAppend(v any) { + // per https://github.com/cloudquery/plugin-sdk/issues/622 + data, err := json.MarshalWithOption(v, json.DisableHTMLEscape()) + if err != nil { + panic(err) + } + + b.ExtensionBuilder.Builder.(*array.BinaryBuilder).UnsafeAppend(data) +} + +func (b *JSONBuilder) AppendValueFromString(s string) error { + if s == array.NullValueStr { + b.AppendNull() + return nil + } + return b.UnmarshalOne(json.NewDecoder(strings.NewReader(s))) +} + +func (b *JSONBuilder) AppendValues(v []any, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + data := make([][]byte, len(v)) + var err error + for i := range v { + if len(valid) > 0 && !valid[i] { + continue + } + // per https://github.com/cloudquery/plugin-sdk/issues/622 + data[i], err = json.MarshalWithOption(v[i], json.DisableHTMLEscape()) + if err != nil { + panic(err) + } + } + b.ExtensionBuilder.Builder.(*array.BinaryBuilder).AppendValues(data, valid) +} + +func (b *JSONBuilder) UnmarshalOne(dec *json.Decoder) error { + var buf any + err := dec.Decode(&buf) + if err != nil { + return err + } + if buf == nil { + b.AppendNull() + } else { + b.Append(buf) + } + return nil +} + +func (b *JSONBuilder) Unmarshal(dec *json.Decoder) error { + for dec.More() { + if err := b.UnmarshalOne(dec); err != nil { + return err + } + } + return nil +} + +func (b *JSONBuilder) UnmarshalJSON(data []byte) error { + dec := json.NewDecoder(bytes.NewReader(data)) + t, err := dec.Token() + if err != nil { + return err + } + + if delim, ok := t.(json.Delim); !ok || delim != '[' { + return fmt.Errorf("JSON builder must unpack from JSON array, found %s", delim) + } + + return b.Unmarshal(dec) +} + +func (b *JSONBuilder) NewJSONArray() *JSONArray { + return b.NewExtensionArray().(*JSONArray) +} + +// JSONArray is a simple array which is a Binary +type JSONArray struct { + array.ExtensionArrayBase +} + +func (a *JSONArray) String() string { + arr := a.Storage().(*array.Binary) + o := new(strings.Builder) + o.WriteString("[") + for i := 0; i < arr.Len(); i++ { + if i > 0 { + o.WriteString(" ") + } + switch { + case a.IsNull(i): + o.WriteString(array.NullValueStr) + default: + fmt.Fprintf(o, "%q", a.ValueStr(i)) + } + } + o.WriteString("]") + return o.String() +} + +func (a *JSONArray) Value(i int) any { + if a.IsNull(i) { + return nil + } + + var data any + err := json.Unmarshal(a.Storage().(*array.Binary).Value(i), &data) + if err != nil { + panic(fmt.Errorf("invalid json: %w", err)) + } + return data +} + +func (a *JSONArray) ValueStr(i int) string { + switch { + case a.IsNull(i): + return array.NullValueStr + default: + return string(a.GetOneForMarshal(i).(json.RawMessage)) + } +} + +func (a *JSONArray) MarshalJSON() ([]byte, error) { + values := make([]json.RawMessage, a.Len()) + for i := 0; i < a.Len(); i++ { + if a.IsNull(i) { + continue + } + values[i] = a.GetOneForMarshal(i).(json.RawMessage) + } + // per https://github.com/cloudquery/plugin-sdk/issues/622 + return json.MarshalWithOption(values, json.DisableHTMLEscape()) +} + +func (a *JSONArray) GetOneForMarshal(i int) any { + if a.IsNull(i) { + return nil + } + return json.RawMessage(a.Storage().(*array.Binary).Value(i)) +} + +// JSONType is a simple extension type that represents a BinaryType +// to be used for representing JSONs +type JSONType struct { + arrow.ExtensionBase +} + +// NewJSONType is a convenience function to create an instance of JSONType +// with the correct storage type +func NewJSONType() *JSONType { + return &JSONType{ExtensionBase: arrow.ExtensionBase{Storage: &arrow.BinaryType{}}} +} + +// ArrayType returns TypeOf(JSONArray{}) for constructing JSON arrays +func (*JSONType) ArrayType() reflect.Type { + return reflect.TypeOf(JSONArray{}) +} + +func (*JSONType) ExtensionName() string { + return "json" +} + +func (*JSONType) String() string { + return "json" +} + +func (e *JSONType) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil +} + +// Serialize returns "json-serialized" for testing proper metadata passing +func (*JSONType) Serialize() string { + return "json-serialized" +} + +// Deserialize expects storageType to be BinaryBuilder and the data to be +// "json-serialized" in order to correctly create a JSONType for testing deserialize. +func (*JSONType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) { + if data != "json-serialized" { + return nil, fmt.Errorf("type identifier did not match: '%s'", data) + } + if !arrow.TypeEqual(storageType, &arrow.BinaryType{}) { + return nil, fmt.Errorf("invalid storage type for *JSONType: %s", storageType.Name()) + } + return NewJSONType(), nil +} + +// ExtensionEquals returns true if both extensions have the same name +func (e *JSONType) ExtensionEquals(other arrow.ExtensionType) bool { + return e.ExtensionName() == other.ExtensionName() +} + +func (*JSONType) NewBuilder(bldr *array.ExtensionBuilder) array.Builder { + return NewJSONBuilder(bldr) +} diff --git a/plugin-sdk/types/json_test.go b/plugin-sdk/types/json_test.go new file mode 100644 index 00000000..364775f0 --- /dev/null +++ b/plugin-sdk/types/json_test.go @@ -0,0 +1,427 @@ +package types + +import ( + "bytes" + "strconv" + "testing" + + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/goccy/go-json" + "github.com/stretchr/testify/require" +) + +func TestJSONBuilder(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + + b := NewJSONBuilder(array.NewExtensionBuilder(mem, NewJSONType())) + b.Append(map[string]any{"a": 1, "b": 2}) + b.AppendNull() + b.Append(map[string]any{"c": 3, "d": 4}) + b.AppendNull() + + require.Equal(t, 4, b.Len(), "unexpected Len()") + require.Equal(t, 2, b.NullN(), "unexpected NullN()") + + values := []any{ + map[string]any{"e": 5, "f": 6}, + map[string]any{"g": 7, "h": 8}, + } + b.AppendValues(values, nil) + + require.Equal(t, 6, b.Len(), "unexpected Len()") + + a := b.NewArray() + + // check state of builder after NewJSONBuilder + require.Zero(t, b.Len(), "unexpected ArrayBuilder.Len(), NewJSONBuilder did not reset state") + require.Zero(t, b.Cap(), "unexpected ArrayBuilder.Cap(), NewJSONBuilder did not reset state") + require.Zero(t, b.NullN(), "unexpected ArrayBuilder.NullN(), NewJSONBuilder did not reset state") + require.Equal(t, `["{\"a\":1,\"b\":2}" (null) "{\"c\":3,\"d\":4}" (null) "{\"e\":5,\"f\":6}" "{\"g\":7,\"h\":8}"]`, a.String()) + st, err := a.MarshalJSON() + require.NoError(t, err) + + b.Release() + a.Release() + + b = NewJSONBuilder(array.NewExtensionBuilder(mem, NewJSONType())) + err = b.UnmarshalJSON(st) + require.NoError(t, err) + + a = b.NewArray() + require.Equal(t, `["{\"a\":1,\"b\":2}" (null) "{\"c\":3,\"d\":4}" (null) "{\"e\":5,\"f\":6}" "{\"g\":7,\"h\":8}"]`, a.String()) + b.Release() + a.Release() +} + +func TestJSONBuilder_UnmarshalOne(t *testing.T) { + cases := []struct { + name string + data string + want string + }{ + { + name: `map`, + data: `{"a": 1, "b": 2}`, + want: `["{\"a\":1,\"b\":2}"]`, + }, + { + name: `two maps`, + data: `{"a": 1, "b": 2}{"c": 3, "d": 4}`, + want: `["{\"a\":1,\"b\":2}"]`, + }, + { + name: `array`, + data: `[1, 2, 3]`, + want: `["[1,2,3]"]`, + }, + { + name: `two arrays`, + data: `[1, 2, 3][4, 5, 6]`, + want: `["[1,2,3]"]`, + }, + { + name: `null`, + data: `null`, + want: `[(null)]`, + }, + { + name: `escaped`, + data: `{"MyKey":"A\u0026B"}`, + want: `["{\"MyKey\":\"A&B\"}"]`, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + b := NewJSONBuilder(array.NewExtensionBuilder(mem, NewJSONType())) + defer b.Release() + dec := json.NewDecoder(bytes.NewReader([]byte(tc.data))) + err := b.UnmarshalOne(dec) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + a := b.NewArray() + defer a.Release() + require.Equal(t, tc.want, a.String()) + }) + } +} + +func TestJSONArray_GetOneForMarshal(t *testing.T) { + cases := []struct { + name string + data string + want json.RawMessage + nil bool + }{ + { + name: `map`, + data: `{"a": 1, "b": 2}`, + want: json.RawMessage(`{"a":1,"b":2}`), + }, + { + name: `two maps`, + data: `{"a": 1, "b": 2}{"c": 3, "d": 4}`, + want: json.RawMessage(`{"a":1,"b":2}`), + }, + { + name: `array`, + data: `[1, 2, 3]`, + want: json.RawMessage(`[1,2,3]`), + }, + { + name: `two arrays`, + data: `[1, 2, 3][4, 5, 6]`, + want: json.RawMessage(`[1,2,3]`), + }, + { + name: `null`, + data: `null`, + nil: true, + }, + { + name: `escaped`, + data: `{"MyKey":"A\u0026B"}`, + want: json.RawMessage(`{"MyKey":"A&B"}`), + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + b := NewJSONBuilder(array.NewExtensionBuilder(mem, NewJSONType())) + defer b.Release() + dec := json.NewDecoder(bytes.NewReader([]byte(tc.data))) + err := b.UnmarshalOne(dec) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + a := b.NewArray() + defer a.Release() + if tc.nil { + require.Nil(t, a.GetOneForMarshal(0)) + } else { + require.Exactly(t, tc.want, a.GetOneForMarshal(0)) + } + }) + } +} + +func TestJSONArray_ValueStrParse(t *testing.T) { + cases := []struct { + name string + data string + want string + nil bool + }{ + { + name: `map`, + data: `{"a": 1, "b": 2}`, + want: `{"a":1,"b":2}`, + }, + { + name: `two maps`, + data: `{"a": 1, "b": 2}{"c": 3, "d": 4}`, + want: `{"a":1,"b":2}`, + }, + { + name: `array`, + data: `[1, 2, 3]`, + want: `[1,2,3]`, + }, + { + name: `two arrays`, + data: `[1, 2, 3][4, 5, 6]`, + want: `[1,2,3]`, + }, + { + name: `null`, + data: `null`, + want: array.NullValueStr, + }, + { + name: `escaped`, + data: `{"MyKey":"A\u0026B"}`, + want: `{"MyKey":"A&B"}`, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + b := NewJSONBuilder(array.NewExtensionBuilder(mem, NewJSONType())) + defer b.Release() + dec := json.NewDecoder(bytes.NewReader([]byte(tc.data))) + err := b.UnmarshalOne(dec) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + a := b.NewArray() + defer a.Release() + require.Exactly(t, tc.want, a.ValueStr(0)) + }) + } +} + +func TestJSONArray_Value(t *testing.T) { + cases := []struct { + name string + data string + want any + }{ + { + name: `map`, + data: `{"a": 1, "b": 2}`, + want: map[string]any{"a": float64(1), "b": float64(2)}, + }, + { + name: `two maps`, + data: `{"a": 1, "b": 2}{"c": 3, "d": 4}`, + want: map[string]any{"a": float64(1), "b": float64(2)}, + }, + { + name: `array`, + data: `[1, 2, 3]`, + want: []any{float64(1), float64(2), float64(3)}, + }, + { + name: `two arrays`, + data: `[1, 2, 3][4, 5, 6]`, + want: []any{float64(1), float64(2), float64(3)}, + }, + { + name: `null`, + data: `null`, + }, + { + name: `escaped`, + data: `[{"MyKey":"A\u0026B"}]`, + want: []any{map[string]any{"MyKey": "A&B"}}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + b := NewJSONBuilder(array.NewExtensionBuilder(mem, NewJSONType())) + defer b.Release() + dec := json.NewDecoder(bytes.NewReader([]byte(tc.data))) + err := b.UnmarshalOne(dec) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + a := b.NewArray().(*JSONArray) + defer a.Release() + require.Equal(t, tc.want, a.Value(0)) + }) + } +} + +func TestJSON_MarshalUnmarshal(t *testing.T) { + cases := []struct { + name string + data string + want []any + }{ + { + name: `map`, + data: `[{"a":1,"b":2}]`, + want: []any{map[string]any{"a": float64(1), "b": float64(2)}}, + }, + { + name: `array`, + data: `[[1,2,3]]`, + want: []any{[]any{float64(1), float64(2), float64(3)}}, + }, + { + name: `empty`, + data: `[]`, + }, + { + name: `mixed`, + data: `[{"a":1,"b":2},null,{"c":3,"d":4},null,{"e":5,"f":6},{"g":7,"h":8}]`, + want: []any{ + map[string]any{"a": float64(1), "b": float64(2)}, + nil, + map[string]any{"c": float64(3), "d": float64(4)}, + nil, + map[string]any{"e": float64(5), "f": float64(6)}, + map[string]any{"g": float64(7), "h": float64(8)}, + }, + }, + { + name: `escaped`, + data: `[{"MyKey":"A\u0026B"}]`, + want: []any{map[string]any{"MyKey": "A&B"}}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + b := NewJSONBuilder(array.NewExtensionBuilder(mem, NewJSONType())) + defer b.Release() + require.NoError(t, b.UnmarshalJSON([]byte(tc.data))) + a := b.NewArray().(*JSONArray) + defer a.Release() + data, err := a.MarshalJSON() + require.NoError(t, err) + require.Equal(t, unescape(tc.data), string(data)) + require.Equal(t, len(tc.want), a.Len()) + for i, elem := range tc.want { + require.Equal(t, elem, a.Value(i)) + } + }) + } +} + +func TestJSON_FromToString(t *testing.T) { + cases := []struct { + name string + data []string + want []any + }{ + { + name: `map`, + data: []string{`{"a":1,"b":2}`}, + want: []any{map[string]any{"a": float64(1), "b": float64(2)}}, + }, + { + name: `array`, + data: []string{`[1,2,3]`}, + want: []any{[]any{float64(1), float64(2), float64(3)}}, + }, + { + name: `empty`, + data: []string{`(null)`}, + want: []any{nil}, + }, + { + name: `mixed`, + data: []string{`{"a":1,"b":2}`, `(null)`, `{"c":3,"d":4}`, `(null)`, `{"e":5,"f":6}`, `{"g":7,"h":8}`}, + want: []any{ + map[string]any{"a": float64(1), "b": float64(2)}, + nil, + map[string]any{"c": float64(3), "d": float64(4)}, + nil, + map[string]any{"e": float64(5), "f": float64(6)}, + map[string]any{"g": float64(7), "h": float64(8)}, + }, + }, + { + name: `escaped`, + data: []string{`{"MyKey":"A\u0026B"}`}, + want: []any{map[string]any{"MyKey": "A&B"}}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + require.Equalf(t, len(tc.want), len(tc.data), "want and data should be of the same length") + + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + b := NewJSONBuilder(array.NewExtensionBuilder(mem, NewJSONType())) + defer b.Release() + for _, str := range tc.data { + require.NoError(t, b.AppendValueFromString(str)) + } + a := b.NewArray().(*JSONArray) + defer a.Release() + + require.Equal(t, len(tc.want), a.Len()) + for i, elem := range tc.want { + require.Equal(t, elem, a.Value(i)) + require.Equal(t, unescape(tc.data[i]), a.ValueStr(i)) + } + }) + } +} + +func unescape(str string) string { + out := "" + for len(str) > 0 { + if str[0] == '\\' { + if len(str) > 5 && str[1] == 'u' { + u, err := strconv.ParseUint(str[2:6], 16, 64) + if err == nil { + out += string(byte(u)) + str = str[6:] + continue + } + } + } + out += str[:1] + str = str[1:] + } + + return out +} diff --git a/plugin-sdk/types/mac.go b/plugin-sdk/types/mac.go new file mode 100644 index 00000000..bb7600cd --- /dev/null +++ b/plugin-sdk/types/mac.go @@ -0,0 +1,234 @@ +package types + +import ( + "bytes" + "fmt" + "net" + "reflect" + "strings" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/goccy/go-json" +) + +type MACBuilder struct { + *array.ExtensionBuilder +} + +func NewMACBuilder(builder *array.ExtensionBuilder) *MACBuilder { + return &MACBuilder{ExtensionBuilder: builder} +} + +func (b *MACBuilder) Append(v net.HardwareAddr) { + b.ExtensionBuilder.Builder.(*array.BinaryBuilder).Append(v[:]) +} + +func (b *MACBuilder) UnsafeAppend(v net.HardwareAddr) { + b.ExtensionBuilder.Builder.(*array.BinaryBuilder).UnsafeAppend(v[:]) +} + +func (b *MACBuilder) AppendValues(v []net.HardwareAddr, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + data := make([][]byte, len(v)) + for i, v := range v { + if len(valid) > 0 && !valid[i] { + continue + } + data[i] = v + } + b.ExtensionBuilder.Builder.(*array.BinaryBuilder).AppendValues(data, valid) +} + +func (b *MACBuilder) AppendValueFromString(s string) error { + if s == array.NullValueStr { + b.AppendNull() + return nil + } + data, err := net.ParseMAC(s) + if err != nil { + return err + } + b.Append(data) + return nil +} + +func (b *MACBuilder) UnmarshalOne(dec *json.Decoder) error { + t, err := dec.Token() + if err != nil { + return err + } + + var val net.HardwareAddr + switch v := t.(type) { + case string: + data, err := net.ParseMAC(v) + if err != nil { + return err + } + val = data + case []byte: + val = net.HardwareAddr(v) + case nil: + b.AppendNull() + return nil + default: + return &json.UnmarshalTypeError{ + Value: fmt.Sprint(t), + Type: reflect.TypeOf([]byte{}), + Offset: dec.InputOffset(), + Struct: "Binary", + } + } + + b.Append(val) + return nil +} + +func (b *MACBuilder) Unmarshal(dec *json.Decoder) error { + for dec.More() { + if err := b.UnmarshalOne(dec); err != nil { + return err + } + } + return nil +} + +func (b *MACBuilder) UnmarshalJSON(data []byte) error { + dec := json.NewDecoder(bytes.NewReader(data)) + t, err := dec.Token() + if err != nil { + return err + } + + if delim, ok := t.(json.Delim); !ok || delim != '[' { + return fmt.Errorf("fixed size binary builder must unpack from json array, found %s", delim) + } + + return b.Unmarshal(dec) +} + +func (b *MACBuilder) NewMACArray() *MACArray { + return b.NewExtensionArray().(*MACArray) +} + +// MACArray is a simple array which is a wrapper around a BinaryArray +type MACArray struct { + array.ExtensionArrayBase +} + +func (a *MACArray) String() string { + arr := a.Storage().(*array.Binary) + o := new(strings.Builder) + o.WriteString("[") + for i := 0; i < arr.Len(); i++ { + if i > 0 { + o.WriteString(" ") + } + switch { + case a.IsNull(i): + o.WriteString(array.NullValueStr) + default: + fmt.Fprintf(o, "%q", a.Value(i)) + } + } + o.WriteString("]") + return o.String() +} + +func (a *MACArray) Value(i int) net.HardwareAddr { + if a.IsNull(i) { + return nil + } + b := a.Storage().(*array.Binary).Value(i) + if len(b) == 0 { + const minMACLen = 6 + return make(net.HardwareAddr, minMACLen) + } + + return net.HardwareAddr(b) +} + +func (a *MACArray) ValueStr(i int) string { + switch { + case a.IsNull(i): + return array.NullValueStr + default: + return a.Value(i).String() + } +} + +func (a *MACArray) MarshalJSON() ([]byte, error) { + arr := a.Storage().(*array.Binary) + values := make([]any, a.Len()) + for i := 0; i < a.Len(); i++ { + if a.IsValid(i) { + values[i] = net.HardwareAddr(arr.Value(i)).String() + } else { + values[i] = nil + } + } + return json.Marshal(values) +} + +func (a *MACArray) GetOneForMarshal(i int) any { + arr := a.Storage().(*array.Binary) + if a.IsValid(i) { + return net.HardwareAddr(arr.Value(i)).String() + } + return nil +} + +// MACType is a simple extension type that represents a BinaryType +// to be used for representing MAC addresses. +type MACType struct { + arrow.ExtensionBase +} + +// NewMACType is a convenience function to create an instance of MACType +// with the correct storage type +func NewMACType() *MACType { + return &MACType{ExtensionBase: arrow.ExtensionBase{Storage: &arrow.BinaryType{}}} +} + +// ArrayType returns TypeOf(MACArray{}) for constructing MAC arrays +func (*MACType) ArrayType() reflect.Type { + return reflect.TypeOf(MACArray{}) +} + +func (*MACType) ExtensionName() string { + return "mac" +} + +func (*MACType) String() string { + return "mac" +} + +// Serialize returns "mac-serialized" for testing proper metadata passing +func (*MACType) Serialize() string { + return "mac-serialized" +} + +// Deserialize expects storageType to be FixedSizeBinaryType{ByteWidth: 16} and the data to be +// "MAC-serialized" in order to correctly create a MACType for testing deserialize. +func (*MACType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) { + if data != "mac-serialized" { + return nil, fmt.Errorf("type identifier did not match: '%s'", data) + } + if !arrow.TypeEqual(storageType, &arrow.BinaryType{}) { + return nil, fmt.Errorf("invalid storage type for MACType: %s", storageType.Name()) + } + return NewMACType(), nil +} + +// ExtensionEquals returns true if both extensions have the same name +func (u *MACType) ExtensionEquals(other arrow.ExtensionType) bool { + return u.ExtensionName() == other.ExtensionName() +} + +func (*MACType) NewBuilder(bldr *array.ExtensionBuilder) array.Builder { + return NewMACBuilder(bldr) +} diff --git a/plugin-sdk/types/mac_test.go b/plugin-sdk/types/mac_test.go new file mode 100644 index 00000000..6a42085b --- /dev/null +++ b/plugin-sdk/types/mac_test.go @@ -0,0 +1,64 @@ +package types + +import ( + "net" + "testing" + + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/stretchr/testify/require" +) + +func mustParseMAC(s string) net.HardwareAddr { + mac, err := net.ParseMAC(s) + if err != nil { + panic(err) + } + return mac +} + +func TestMACBuilder(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + + b := NewMACBuilder(array.NewExtensionBuilder(mem, NewMACType())) + + b.Append(mustParseMAC("00:00:00:00:00:01")) + b.AppendNull() + b.Append(mustParseMAC("00:00:00:00:00:02")) + b.AppendNull() + + require.Equal(t, 4, b.Len(), "unexpected Len()") + require.Equal(t, 2, b.NullN(), "unexpected NullN()") + + values := []net.HardwareAddr{ + mustParseMAC("00:00:00:00:00:03"), + mustParseMAC("00:00:00:00:00:04"), + } + b.AppendValues(values, nil) + + require.Equal(t, 6, b.Len(), "unexpected Len()") + + a := b.NewArray() + + // check state of builder after NewArray + require.Zero(t, b.Len(), "unexpected ArrayBuilder.Len(), did not reset state") + require.Zero(t, b.Cap(), "unexpected ArrayBuilder.Cap(), did not reset state") + require.Zero(t, b.NullN(), "unexpected ArrayBuilder.NullN(), did not reset state") + + require.Equal(t, `["00:00:00:00:00:01" (null) "00:00:00:00:00:02" (null) "00:00:00:00:00:03" "00:00:00:00:00:04"]`, a.String()) + st, err := a.MarshalJSON() + require.NoError(t, err) + + b.Release() + a.Release() + + b = NewMACBuilder(array.NewExtensionBuilder(mem, NewMACType())) + err = b.UnmarshalJSON(st) + require.NoError(t, err) + + a = b.NewArray() + require.Equal(t, `["00:00:00:00:00:01" (null) "00:00:00:00:00:02" (null) "00:00:00:00:00:03" "00:00:00:00:00:04"]`, a.String()) + b.Release() + a.Release() +} diff --git a/plugin-sdk/types/register.go b/plugin-sdk/types/register.go new file mode 100644 index 00000000..68669395 --- /dev/null +++ b/plugin-sdk/types/register.go @@ -0,0 +1,29 @@ +package types + +import "github.com/apache/arrow/go/v16/arrow" + +func RegisterAllExtensions() error { + if err := arrow.RegisterExtensionType(&UUIDType{}); err != nil { + return err + } + if err := arrow.RegisterExtensionType(&JSONType{}); err != nil { + return err + } + if err := arrow.RegisterExtensionType(&InetType{}); err != nil { + return err + } + return arrow.RegisterExtensionType(&MACType{}) +} + +func UnregisterAllExtensions() error { + if err := arrow.UnregisterExtensionType(ExtensionTypes.MAC.ExtensionName()); err != nil { + return err + } + if err := arrow.UnregisterExtensionType(ExtensionTypes.Inet.ExtensionName()); err != nil { + return err + } + if err := arrow.UnregisterExtensionType(ExtensionTypes.JSON.ExtensionName()); err != nil { + return err + } + return arrow.UnregisterExtensionType(ExtensionTypes.UUID.ExtensionName()) +} diff --git a/plugin-sdk/types/uuid.go b/plugin-sdk/types/uuid.go new file mode 100644 index 00000000..2aacc2f1 --- /dev/null +++ b/plugin-sdk/types/uuid.go @@ -0,0 +1,235 @@ +package types + +import ( + "bytes" + "fmt" + "reflect" + "strings" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/goccy/go-json" + "github.com/google/uuid" +) + +var UUID = NewUUIDType() + +type UUIDBuilder struct { + *array.ExtensionBuilder +} + +func NewUUIDBuilder(builder *array.ExtensionBuilder) *UUIDBuilder { + return &UUIDBuilder{ExtensionBuilder: builder} +} + +func (b *UUIDBuilder) Append(v uuid.UUID) { + b.ExtensionBuilder.Builder.(*array.FixedSizeBinaryBuilder).Append(v[:]) +} + +func (b *UUIDBuilder) UnsafeAppend(v uuid.UUID) { + b.ExtensionBuilder.Builder.(*array.FixedSizeBinaryBuilder).UnsafeAppend(v[:]) +} + +func (b *UUIDBuilder) AppendValueFromString(s string) error { + if s == array.NullValueStr { + b.AppendNull() + return nil + } + + uid, err := uuid.Parse(s) + if err != nil { + return err + } + + b.Append(uid) + return nil +} + +func (b *UUIDBuilder) AppendValues(v []uuid.UUID, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + data := make([][]byte, len(v)) + for i := range v { + if len(valid) > 0 && !valid[i] { + continue + } + data[i] = v[i][:] + } + b.ExtensionBuilder.Builder.(*array.FixedSizeBinaryBuilder).AppendValues(data, valid) +} + +func (b *UUIDBuilder) UnmarshalOne(dec *json.Decoder) error { + t, err := dec.Token() + if err != nil { + return err + } + + var val uuid.UUID + switch v := t.(type) { + case string: + val, err = uuid.Parse(v) + if err != nil { + return err + } + case []byte: + val, err = uuid.ParseBytes(v) + if err != nil { + return err + } + case nil: + b.AppendNull() + return nil + default: + return &json.UnmarshalTypeError{ + Value: fmt.Sprint(t), + Type: reflect.TypeOf([]byte{}), + Offset: dec.InputOffset(), + Struct: fmt.Sprintf("FixedSizeBinary[%d]", 16), + } + } + + b.Append(val) + return nil +} + +func (b *UUIDBuilder) Unmarshal(dec *json.Decoder) error { + for dec.More() { + if err := b.UnmarshalOne(dec); err != nil { + return err + } + } + return nil +} + +func (b *UUIDBuilder) UnmarshalJSON(data []byte) error { + dec := json.NewDecoder(bytes.NewReader(data)) + t, err := dec.Token() + if err != nil { + return err + } + + if delim, ok := t.(json.Delim); !ok || delim != '[' { + return fmt.Errorf("uuid builder must unpack from json array, found %s", delim) + } + + return b.Unmarshal(dec) +} + +func (b *UUIDBuilder) NewUUIDArray() *UUIDArray { + return b.NewExtensionArray().(*UUIDArray) +} + +// UUIDArray is a simple array which is a FixedSizeBinary(16) +type UUIDArray struct { + array.ExtensionArrayBase +} + +func (a *UUIDArray) String() string { + arr := a.Storage().(*array.FixedSizeBinary) + o := new(strings.Builder) + o.WriteString("[") + for i := 0; i < arr.Len(); i++ { + if i > 0 { + o.WriteString(" ") + } + switch { + case a.IsNull(i): + o.WriteString(array.NullValueStr) + default: + fmt.Fprintf(o, "%q", a.Value(i)) + } + } + o.WriteString("]") + return o.String() +} + +func (a *UUIDArray) Value(i int) uuid.UUID { + if a.IsNull(i) { + return uuid.Nil + } + return uuid.Must(uuid.FromBytes(a.Storage().(*array.FixedSizeBinary).Value(i))) +} + +func (a *UUIDArray) ValueStr(i int) string { + switch { + case a.IsNull(i): + return array.NullValueStr + default: + return a.Value(i).String() + } +} + +func (a *UUIDArray) MarshalJSON() ([]byte, error) { + arr := a.Storage().(*array.FixedSizeBinary) + values := make([]any, a.Len()) + for i := 0; i < a.Len(); i++ { + if a.IsValid(i) { + values[i] = uuid.Must(uuid.FromBytes(arr.Value(i))).String() + } + } + return json.Marshal(values) +} + +func (a *UUIDArray) GetOneForMarshal(i int) any { + if a.IsNull(i) { + return nil + } + return a.Value(i) +} + +// UUIDType is a simple extension type that represents a FixedSizeBinary(16) +// to be used for representing UUIDs +type UUIDType struct { + arrow.ExtensionBase +} + +// NewUUIDType is a convenience function to create an instance of UUIDType +// with the correct storage type +func NewUUIDType() *UUIDType { + return &UUIDType{ExtensionBase: arrow.ExtensionBase{Storage: &arrow.FixedSizeBinaryType{ByteWidth: 16}}} +} + +// ArrayType returns TypeOf(UUIDArray{}) for constructing UUID arrays +func (*UUIDType) ArrayType() reflect.Type { + return reflect.TypeOf(UUIDArray{}) +} + +func (*UUIDType) ExtensionName() string { + return "uuid" +} + +func (*UUIDType) String() string { + return "uuid" +} + +func (e *UUIDType) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil +} + +// Serialize returns "uuid-serialized" for testing proper metadata passing +func (*UUIDType) Serialize() string { + return "uuid-serialized" +} + +// Deserialize expects storageType to be FixedSizeBinaryType{ByteWidth: 16} and the data to be +// "uuid-serialized" in order to correctly create a UUIDType for testing deserialize. +func (*UUIDType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) { + if data != "uuid-serialized" { + return nil, fmt.Errorf("type identifier did not match: '%s'", data) + } + if !arrow.TypeEqual(storageType, &arrow.FixedSizeBinaryType{ByteWidth: 16}) { + return nil, fmt.Errorf("invalid storage type for UUIDType: %s", storageType.Name()) + } + return NewUUIDType(), nil +} + +// ExtensionEquals returns true if both extensions have the same name +func (e *UUIDType) ExtensionEquals(other arrow.ExtensionType) bool { + return e.ExtensionName() == other.ExtensionName() +} + +func (*UUIDType) NewBuilder(bldr *array.ExtensionBuilder) array.Builder { + return NewUUIDBuilder(bldr) +} diff --git a/plugin-sdk/types/uuid_test.go b/plugin-sdk/types/uuid_test.go new file mode 100644 index 00000000..0e462593 --- /dev/null +++ b/plugin-sdk/types/uuid_test.go @@ -0,0 +1,56 @@ +package types + +import ( + "testing" + + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func TestUUIDBuilder(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + + b := NewUUIDBuilder(array.NewExtensionBuilder(mem, NewUUIDType())) + + b.Append(uuid.MustParse("00000000-0000-0000-0000-000000000001")) + b.AppendNull() + b.Append(uuid.MustParse("00000000-0000-0000-0000-000000000002")) + b.AppendNull() + + require.Equal(t, 4, b.Len(), "unexpected Len()") + require.Equal(t, 2, b.NullN(), "unexpected NullN()") + + values := []uuid.UUID{ + uuid.MustParse("00000000-0000-0000-0000-000000000003"), + uuid.MustParse("00000000-0000-0000-0000-000000000004"), + } + b.AppendValues(values, nil) + + require.Equal(t, 6, b.Len(), "unexpected Len()") + + a := b.NewArray() + + // check state of builder after NewUUIDBuilder + require.Zero(t, b.Len(), "unexpected ArrayBuilder.Len(), NewUUIDBuilder did not reset state") + require.Zero(t, b.Cap(), "unexpected ArrayBuilder.Cap(), NewUUIDBuilder did not reset state") + require.Zero(t, b.NullN(), "unexpected ArrayBuilder.NullN(), NewUUIDBuilder did not reset state") + + require.Equal(t, `["00000000-0000-0000-0000-000000000001" (null) "00000000-0000-0000-0000-000000000002" (null) "00000000-0000-0000-0000-000000000003" "00000000-0000-0000-0000-000000000004"]`, a.String()) + st, err := a.MarshalJSON() + require.NoError(t, err) + + b.Release() + a.Release() + + b = NewUUIDBuilder(array.NewExtensionBuilder(mem, NewUUIDType())) + err = b.UnmarshalJSON(st) + require.NoError(t, err) + + a = b.NewArray() + require.Equal(t, `["00000000-0000-0000-0000-000000000001" (null) "00000000-0000-0000-0000-000000000002" (null) "00000000-0000-0000-0000-000000000003" "00000000-0000-0000-0000-000000000004"]`, a.String()) + b.Release() + a.Release() +} diff --git a/plugin-sdk/writers/batchwriter/batchwriter.go b/plugin-sdk/writers/batchwriter/batchwriter.go new file mode 100644 index 00000000..2da61da1 --- /dev/null +++ b/plugin-sdk/writers/batchwriter/batchwriter.go @@ -0,0 +1,368 @@ +package batchwriter + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/rs/zerolog" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/internal/batch" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/writers" +) + +type Client interface { + MigrateTables(context.Context, message.WriteMigrateTables) error + WriteTableBatch(ctx context.Context, name string, messages message.WriteInserts) error + DeleteStale(context.Context, message.WriteDeleteStales) error + DeleteRecord(context.Context, message.WriteDeleteRecords) error +} + +type BatchWriter struct { + client Client + workers map[string]*worker + workersLock sync.RWMutex + workersWaitGroup sync.WaitGroup + + migrateTableLock sync.Mutex + migrateTableMessages message.WriteMigrateTables + deleteStaleLock sync.Mutex + deleteStaleMessages message.WriteDeleteStales + deleteRecordLock sync.Mutex + deleteRecordMessages message.WriteDeleteRecords + + logger zerolog.Logger + batchTimeout time.Duration + batchSize int64 + batchSizeBytes int64 +} + +// Assert at compile-time that BatchWriter implements the Writer interface +var _ writers.Writer = (*BatchWriter)(nil) + +type Option func(*BatchWriter) + +func WithLogger(logger zerolog.Logger) Option { + return func(p *BatchWriter) { + p.logger = logger + } +} + +func WithBatchTimeout(timeout time.Duration) Option { + return func(p *BatchWriter) { + p.batchTimeout = timeout + } +} + +func WithBatchSize(size int64) Option { + return func(p *BatchWriter) { + p.batchSize = size + } +} + +func WithBatchSizeBytes(size int64) Option { + return func(p *BatchWriter) { + p.batchSizeBytes = size + } +} + +type worker struct { + ch chan *message.WriteInsert + flush chan chan bool +} + +const ( + defaultBatchTimeoutSeconds = 20 + defaultBatchSize = 10000 + defaultBatchSizeBytes = 5 * 1024 * 1024 // 5 MiB +) + +func New(client Client, opts ...Option) (*BatchWriter, error) { + c := &BatchWriter{ + client: client, + workers: make(map[string]*worker), + logger: zerolog.Nop(), + batchTimeout: defaultBatchTimeoutSeconds * time.Second, + batchSize: defaultBatchSize, + batchSizeBytes: defaultBatchSizeBytes, + } + for _, opt := range opts { + opt(c) + } + c.migrateTableMessages = make([]*message.WriteMigrateTable, 0, c.batchSize) + c.deleteStaleMessages = make([]*message.WriteDeleteStale, 0, c.batchSize) + return c, nil +} + +func (w *BatchWriter) Flush(ctx context.Context) error { + w.workersLock.RLock() + for _, worker := range w.workers { + done := make(chan bool) + worker.flush <- done + <-done + } + w.workersLock.RUnlock() + if err := w.flushMigrateTables(ctx); err != nil { + return err + } + return w.flushDeleteStaleTables(ctx) +} + +func (w *BatchWriter) Close(context.Context) error { + w.workersLock.Lock() + defer w.workersLock.Unlock() + for _, w := range w.workers { + close(w.ch) + } + w.workersWaitGroup.Wait() + + return nil +} + +func (w *BatchWriter) worker(ctx context.Context, tableName string, ch <-chan *message.WriteInsert, flush <-chan chan bool) { + limit := batch.CappedAt(w.batchSizeBytes, w.batchSize) + resources := make(message.WriteInserts, 0, w.batchSize) // at least we have 1 row per record + + ticker := writers.NewTicker(w.batchTimeout) + defer ticker.Stop() + + tickerCh, ctxDone := ticker.Chan(), ctx.Done() + + send := func() { + w.flushTable(ctx, tableName, resources, limit) + clear(resources) + resources = resources[:0] + limit.Reset() + } + + for { + select { + case r, ok := <-ch: + if !ok { + if limit.Rows() > 0 { + w.flushTable(ctx, tableName, resources, limit) + } + return + } + + if r.Record.NumRows() == 0 { + // skip empty ones + continue + } + + add, toFlush, rest := batch.SliceRecord(r.Record, limit) + if add != nil { + resources = append(resources, &message.WriteInsert{Record: add.Record}) + limit.AddSlice(add) + } + if len(toFlush) > 0 || rest != nil || limit.ReachedLimit() { + // flush current batch + send() + ticker.Reset(w.batchTimeout) + } + for _, sliceToFlush := range toFlush { + resources = append(resources, &message.WriteInsert{Record: sliceToFlush}) + limit.AddRows(sliceToFlush.NumRows()) + send() + ticker.Reset(w.batchTimeout) + } + + // set the remainder + if rest != nil { + resources = append(resources, &message.WriteInsert{Record: rest.Record}) + limit.AddSlice(rest) + } + + case <-tickerCh: + if limit.Rows() > 0 { + send() + } + case done := <-flush: + if limit.Rows() > 0 { + send() + ticker.Reset(w.batchTimeout) + } + done <- true + case <-ctxDone: + // this means the request was cancelled + return // after this NO other call will succeed + } + } +} + +func (w *BatchWriter) flushTable(ctx context.Context, tableName string, resources message.WriteInserts, limit *batch.Cap) { + batchSize := limit.Rows() + start := time.Now() + err := w.client.WriteTableBatch(ctx, tableName, resources) + duration := time.Since(start) + if err != nil { + w.logger.Err(err).Str("table", tableName).Int64("len", batchSize).Dur("duration", duration).Msg("failed to write batch") + } else { + w.logger.Debug().Str("table", tableName).Int64("len", batchSize).Dur("duration", duration).Msg("batch written successfully") + } +} + +func (w *BatchWriter) flushMigrateTables(ctx context.Context) error { + w.migrateTableLock.Lock() + defer w.migrateTableLock.Unlock() + if len(w.migrateTableMessages) == 0 { + return nil + } + if err := w.client.MigrateTables(ctx, w.migrateTableMessages); err != nil { + return err + } + w.migrateTableMessages = w.migrateTableMessages[:0] + return nil +} + +func (w *BatchWriter) flushDeleteStaleTables(ctx context.Context) error { + w.deleteStaleLock.Lock() + defer w.deleteStaleLock.Unlock() + if len(w.deleteStaleMessages) == 0 { + return nil + } + if err := w.client.DeleteStale(ctx, w.deleteStaleMessages); err != nil { + return err + } + w.deleteStaleMessages = w.deleteStaleMessages[:0] + return nil +} + +func (w *BatchWriter) flushDeleteRecordTables(ctx context.Context) error { + w.deleteRecordLock.Lock() + defer w.deleteRecordLock.Unlock() + if len(w.deleteRecordMessages) == 0 { + return nil + } + if err := w.client.DeleteRecord(ctx, w.deleteRecordMessages); err != nil { + return err + } + w.deleteRecordMessages = w.deleteRecordMessages[:0] + return nil +} + +func (w *BatchWriter) flushInsert(tableName string) { + w.workersLock.RLock() + worker, ok := w.workers[tableName] + if !ok { + w.workersLock.RUnlock() + // no tables to flush + return + } + w.workersLock.RUnlock() + ch := make(chan bool) + worker.flush <- ch + <-ch +} + +func (w *BatchWriter) writeAll(ctx context.Context, msgs []message.WriteMessage) error { + ch := make(chan message.WriteMessage, len(msgs)) + for _, msg := range msgs { + ch <- msg + } + close(ch) + return w.Write(ctx, ch) +} + +func (w *BatchWriter) Write(ctx context.Context, msgs <-chan message.WriteMessage) error { + for msg := range msgs { + switch m := msg.(type) { + case *message.WriteDeleteStale: + if err := w.flushMigrateTables(ctx); err != nil { + return err + } + w.flushInsert(m.TableName) + w.deleteStaleLock.Lock() + w.deleteStaleMessages = append(w.deleteStaleMessages, m) + l := int64(len(w.deleteStaleMessages)) + w.deleteStaleLock.Unlock() + if w.batchSize > 0 && l > w.batchSize { + if err := w.flushDeleteStaleTables(ctx); err != nil { + return err + } + } + case *message.WriteDeleteRecord: + if err := w.flushMigrateTables(ctx); err != nil { + return err + } + if err := w.flushDeleteStaleTables(ctx); err != nil { + return err + } + // Ensure all related workers are flushed + for _, rel := range m.TableRelations { + w.flushInsert(rel.TableName) + } + w.deleteRecordLock.Lock() + w.deleteRecordMessages = append(w.deleteRecordMessages, m) + l := int64(len(w.deleteRecordMessages)) + w.deleteRecordLock.Unlock() + if w.batchSize > 0 && l > w.batchSize { + if err := w.flushDeleteRecordTables(ctx); err != nil { + return err + } + } + case *message.WriteInsert: + if err := w.flushMigrateTables(ctx); err != nil { + return err + } + if err := w.flushDeleteStaleTables(ctx); err != nil { + return err + } + if err := w.startWorker(ctx, m); err != nil { + return err + } + case *message.WriteMigrateTable: + w.flushInsert(m.Table.Name) + if err := w.flushDeleteStaleTables(ctx); err != nil { + return err + } + w.migrateTableLock.Lock() + w.migrateTableMessages = append(w.migrateTableMessages, m) + l := int64(len(w.migrateTableMessages)) + w.migrateTableLock.Unlock() + if w.batchSize > 0 && l > w.batchSize { + if err := w.flushMigrateTables(ctx); err != nil { + return err + } + } + } + } + return nil +} + +func (w *BatchWriter) startWorker(_ context.Context, msg *message.WriteInsert) error { + w.workersLock.RLock() + md := msg.Record.Schema().Metadata() + tableName, ok := md.GetValue(schema.MetadataTableName) + if !ok { + w.workersLock.RUnlock() + return fmt.Errorf("table name not found in metadata") + } + wr, ok := w.workers[tableName] + w.workersLock.RUnlock() + if ok { + wr.ch <- msg + return nil + } + w.workersLock.Lock() + ch := make(chan *message.WriteInsert) + flush := make(chan chan bool) + wr = &worker{ + ch: ch, + flush: flush, + } + w.workers[tableName] = wr + w.workersLock.Unlock() + w.workersWaitGroup.Add(1) + go func() { + defer w.workersWaitGroup.Done() + // TODO: we need to create a cancellable context that then can be cancelled via + // w.cancelWorkers() + w.worker(context.Background(), tableName, ch, flush) + }() + ch <- msg + return nil +} diff --git a/plugin-sdk/writers/batchwriter/batchwriter_test.go b/plugin-sdk/writers/batchwriter/batchwriter_test.go new file mode 100644 index 00000000..2071f60c --- /dev/null +++ b/plugin-sdk/writers/batchwriter/batchwriter_test.go @@ -0,0 +1,247 @@ +package batchwriter + +import ( + "context" + "math/rand" + "strconv" + "sync" + "testing" + "time" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +type testBatchClient struct { + mutex sync.Mutex + migrateTables message.WriteMigrateTables + inserts message.WriteInserts + deleteStales message.WriteDeleteStales + deleteRecords message.WriteDeleteRecords +} + +func (c *testBatchClient) MigrateTablesLen() int { + c.mutex.Lock() + defer c.mutex.Unlock() + return len(c.migrateTables) +} + +func (c *testBatchClient) InsertsLen() int { + c.mutex.Lock() + defer c.mutex.Unlock() + return len(c.inserts) +} + +func (c *testBatchClient) DeleteStalesLen() int { + c.mutex.Lock() + defer c.mutex.Unlock() + return len(c.deleteStales) +} + +func (c *testBatchClient) MigrateTables(_ context.Context, messages message.WriteMigrateTables) error { + c.mutex.Lock() + defer c.mutex.Unlock() + c.migrateTables = append(c.migrateTables, messages...) + return nil +} + +func (c *testBatchClient) WriteTableBatch(_ context.Context, _ string, messages message.WriteInserts) error { + c.mutex.Lock() + defer c.mutex.Unlock() + c.inserts = append(c.inserts, messages...) + return nil +} + +func (c *testBatchClient) DeleteStale(_ context.Context, messages message.WriteDeleteStales) error { + c.mutex.Lock() + defer c.mutex.Unlock() + c.deleteStales = append(c.deleteStales, messages...) + return nil +} + +func (c *testBatchClient) DeleteRecord(_ context.Context, messages message.WriteDeleteRecords) error { + c.mutex.Lock() + defer c.mutex.Unlock() + c.deleteRecords = append(c.deleteRecords, messages...) + return nil +} + +var batchTestTables = schema.Tables{ + { + Name: "table1", + Columns: []schema.Column{ + { + Name: "id", + Type: arrow.PrimitiveTypes.Int64, + }, + }, + }, +} + +// TestBatchFlushDifferentMessages tests that if writer receives a message of a new type all other pending +// batches are flushed. +func TestBatchFlushDifferentMessages(t *testing.T) { + ctx := context.Background() + + testClient := &testBatchClient{} + wr, err := New(testClient) + if err != nil { + t.Fatal(err) + } + + record := getRecord(batchTestTables[0].ToArrowSchema(), 1) + if err := wr.writeAll(ctx, []message.WriteMessage{&message.WriteMigrateTable{Table: batchTestTables[0]}}); err != nil { + t.Fatal(err) + } + + if testClient.MigrateTablesLen() != 0 { + t.Fatalf("expected 0 create table messages, got %d", testClient.MigrateTablesLen()) + } + + if err := wr.writeAll(ctx, []message.WriteMessage{&message.WriteInsert{Record: record}}); err != nil { + t.Fatal(err) + } + + if testClient.MigrateTablesLen() != 1 { + t.Fatalf("expected 1 migrate table message, got %d", testClient.MigrateTablesLen()) + } + + if testClient.InsertsLen() != 0 { + t.Fatalf("expected 0 insert messages, got %d", testClient.InsertsLen()) + } + + if err := wr.writeAll(ctx, []message.WriteMessage{&message.WriteMigrateTable{Table: batchTestTables[0]}}); err != nil { + t.Fatal(err) + } + + if testClient.InsertsLen() != 1 { + t.Fatalf("expected 1 insert message, got %d", testClient.InsertsLen()) + } +} + +func TestBatchSize(t *testing.T) { + for i := 0; i < 5; i++ { + batchSize := rand.Intn(100) + 10 + t.Run(strconv.Itoa(batchSize), func(t *testing.T) { + t.Parallel() + testClient := &testBatchClient{} + wr, err := New(testClient, WithBatchSize(int64(batchSize*2))) + if err != nil { + t.Fatal(err) + } + table := schema.Table{Name: "table1", Columns: []schema.Column{{Name: "id", Type: arrow.PrimitiveTypes.Int64}}} + record := getRecord(table.ToArrowSchema(), batchSize) + if err := wr.writeAll(context.TODO(), []message.WriteMessage{&message.WriteInsert{ + Record: record, + }}); err != nil { + t.Fatal(err) + } + + if testClient.InsertsLen() != 0 { + t.Fatalf("expected 0 insert messages, got %d", testClient.InsertsLen()) + } + + if err := wr.writeAll(context.TODO(), []message.WriteMessage{ + &message.WriteInsert{ + Record: record, + }, + &message.WriteInsert{ // third message to exceed the batch size + Record: record, + }, + }); err != nil { + t.Fatal(err) + } + // we need to wait for the batch to be flushed + time.Sleep(time.Second * 2) + + if testClient.InsertsLen() != 2 { + t.Fatalf("expected 2 insert messages, got %d", testClient.InsertsLen()) + } + }) + } +} + +func TestBatchTimeout(t *testing.T) { + ctx := context.Background() + + testClient := &testBatchClient{} + wr, err := New(testClient, WithBatchTimeout(time.Second)) + if err != nil { + t.Fatal(err) + } + table := schema.Table{Name: "table1", Columns: []schema.Column{{Name: "id", Type: arrow.PrimitiveTypes.Int64}}} + record := getRecord(table.ToArrowSchema(), 1) + if err := wr.writeAll(ctx, []message.WriteMessage{&message.WriteInsert{ + Record: record, + }}); err != nil { + t.Fatal(err) + } + + if testClient.InsertsLen() != 0 { + t.Fatalf("expected 0 insert messages, got %d", testClient.InsertsLen()) + } + + // we need to wait for the batch to be flushed + time.Sleep(time.Millisecond * 250) + + if testClient.InsertsLen() != 0 { + t.Fatalf("expected 0 insert messages, got %d", testClient.InsertsLen()) + } + + // we need to wait for the batch to be flushed + time.Sleep(time.Second * 1) + + if testClient.InsertsLen() != 1 { + t.Fatalf("expected 1 insert message, got %d", testClient.InsertsLen()) + } +} + +func TestBatchUpserts(t *testing.T) { + ctx := context.Background() + + testClient := &testBatchClient{} + wr, err := New(testClient, WithBatchSize(2), WithBatchTimeout(time.Second)) + if err != nil { + t.Fatal(err) + } + table := schema.Table{Name: "table1", Columns: []schema.Column{{Name: "id", Type: arrow.PrimitiveTypes.Int64, PrimaryKey: true}}} + + record := getRecord(table.ToArrowSchema(), 1) + + if err := wr.writeAll(ctx, []message.WriteMessage{&message.WriteInsert{ + Record: record, + }}); err != nil { + t.Fatal(err) + } + + if testClient.InsertsLen() != 0 { + t.Fatalf("expected 0 insert messages, got %d", testClient.InsertsLen()) + } + + if err := wr.writeAll(ctx, []message.WriteMessage{&message.WriteInsert{ + Record: record, + }}); err != nil { + t.Fatal(err) + } + // we need to wait for the batch to be flushed + time.Sleep(time.Second * 2) + + if testClient.InsertsLen() != 2 { + t.Fatalf("expected 2 insert messages, got %d", testClient.InsertsLen()) + } +} + +func getRecord(sc *arrow.Schema, rows int) arrow.Record { + builder := array.NewRecordBuilder(memory.DefaultAllocator, sc) + defer builder.Release() + + for _, f := range builder.Fields() { + f.AppendEmptyValues(rows) + } + + return builder.NewRecord() +} diff --git a/plugin-sdk/writers/batchwriter/unimplemented.go b/plugin-sdk/writers/batchwriter/unimplemented.go new file mode 100644 index 00000000..805882ba --- /dev/null +++ b/plugin-sdk/writers/batchwriter/unimplemented.go @@ -0,0 +1,27 @@ +package batchwriter + +import ( + "context" + "fmt" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/plugin" +) + +type IgnoreMigrateTables struct{} + +func (IgnoreMigrateTables) MigrateTables(context.Context, message.WriteMigrateTables) error { + return nil +} + +type UnimplementedDeleteStale struct{} + +func (UnimplementedDeleteStale) DeleteStale(context.Context, message.WriteDeleteStales) error { + return fmt.Errorf("DeleteStale: %w", plugin.ErrNotImplemented) +} + +type UnimplementedDeleteRecord struct{} + +func (UnimplementedDeleteRecord) DeleteRecord(context.Context, message.WriteDeleteRecords) error { + return fmt.Errorf("DeleteRecord: %w", plugin.ErrNotImplemented) +} diff --git a/plugin-sdk/writers/batchwriter/unimplemented_test.go b/plugin-sdk/writers/batchwriter/unimplemented_test.go new file mode 100644 index 00000000..a86ae673 --- /dev/null +++ b/plugin-sdk/writers/batchwriter/unimplemented_test.go @@ -0,0 +1,20 @@ +package batchwriter_test + +import ( + "context" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/writers/batchwriter" +) + +type testDummyClient struct { + batchwriter.IgnoreMigrateTables + batchwriter.UnimplementedDeleteStale + batchwriter.UnimplementedDeleteRecord +} + +func (testDummyClient) WriteTableBatch(context.Context, string, message.WriteInserts) error { + return nil +} + +var _ batchwriter.Client = (*testDummyClient)(nil) diff --git a/plugin-sdk/writers/mixedbatchwriter/mixedbatchwriter.go b/plugin-sdk/writers/mixedbatchwriter/mixedbatchwriter.go new file mode 100644 index 00000000..c5ce1246 --- /dev/null +++ b/plugin-sdk/writers/mixedbatchwriter/mixedbatchwriter.go @@ -0,0 +1,256 @@ +package mixedbatchwriter + +import ( + "context" + "time" + + "github.com/rs/zerolog" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/internal/batch" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/writers" +) + +// Client is a client that will receive batches of messages with a mixture of tables. +type Client interface { + MigrateTableBatch(ctx context.Context, messages message.WriteMigrateTables) error + InsertBatch(ctx context.Context, messages message.WriteInserts) error + DeleteStaleBatch(ctx context.Context, messages message.WriteDeleteStales) error + DeleteRecordsBatch(ctx context.Context, messages message.WriteDeleteRecords) error +} + +type MixedBatchWriter struct { + client Client + logger zerolog.Logger + batchSize int64 + batchSizeBytes int64 + batchTimeout time.Duration + tickerFn writers.TickerFunc +} + +// Assert at compile-time that MixedBatchWriter implements the Writer interface +var _ writers.Writer = (*MixedBatchWriter)(nil) + +type Option func(writer *MixedBatchWriter) + +func WithLogger(logger zerolog.Logger) Option { + return func(p *MixedBatchWriter) { + p.logger = logger + } +} + +func WithBatchSize(size int64) Option { + return func(p *MixedBatchWriter) { + p.batchSize = size + } +} + +func WithBatchSizeBytes(size int64) Option { + return func(p *MixedBatchWriter) { + p.batchSizeBytes = size + } +} + +func WithBatchTimeout(timeout time.Duration) Option { + return func(p *MixedBatchWriter) { + p.batchTimeout = timeout + } +} + +func withTickerFn(tickerFn writers.TickerFunc) Option { + return func(p *MixedBatchWriter) { + p.tickerFn = tickerFn + } +} + +const ( + defaultBatchTimeout = 20 * time.Second + defaultBatchSize = 10000 + defaultBatchSizeBytes = 5 * 1024 * 1024 // 5 MiB +) + +func New(client Client, opts ...Option) (*MixedBatchWriter, error) { + c := &MixedBatchWriter{ + client: client, + logger: zerolog.Nop(), + batchSize: defaultBatchSize, + batchSizeBytes: defaultBatchSizeBytes, + batchTimeout: defaultBatchTimeout, + tickerFn: writers.NewTicker, + } + for _, opt := range opts { + opt(c) + } + return c, nil +} + +// Write starts listening for messages on the msgChan channel and writes them to the client in batches. +func (w *MixedBatchWriter) Write(ctx context.Context, msgChan <-chan message.WriteMessage) error { + migrateTable := &batchManager[message.WriteMigrateTables, *message.WriteMigrateTable]{ + batch: make([]*message.WriteMigrateTable, 0, w.batchSize), + writeFunc: w.client.MigrateTableBatch, + } + insert := &insertBatchManager{ + batch: make([]*message.WriteInsert, 0, w.batchSize), + writeFunc: w.client.InsertBatch, + limit: batch.CappedAt(w.batchSizeBytes, w.batchSize), + logger: w.logger, + } + deleteStale := &batchManager[message.WriteDeleteStales, *message.WriteDeleteStale]{ + batch: make([]*message.WriteDeleteStale, 0, w.batchSize), + writeFunc: w.client.DeleteStaleBatch, + } + + deleteRecord := &batchManager[message.WriteDeleteRecords, *message.WriteDeleteRecord]{ + batch: make([]*message.WriteDeleteRecord, 0, w.batchSize), + writeFunc: w.client.DeleteRecordsBatch, + } + + flush := func(msgType writers.MsgType) error { + if msgType == writers.MsgTypeUnset { + return nil + } + switch msgType { + case writers.MsgTypeMigrateTable: + return migrateTable.flush(ctx) + case writers.MsgTypeInsert: + return insert.flush(ctx) + case writers.MsgTypeDeleteStale: + return deleteStale.flush(ctx) + case writers.MsgTypeDeleteRecord: + return deleteRecord.flush(ctx) + default: + panic("unknown message type") + } + } + prevMsgType := writers.MsgTypeUnset + var err error + ticker := w.tickerFn(w.batchTimeout) + defer ticker.Stop() +loop: + for { + select { + case msg, ok := <-msgChan: + if !ok { + break loop + } + msgType := writers.MsgID(msg) + if prevMsgType != msgType { + if err := flush(prevMsgType); err != nil { + return err + } + ticker.Reset(w.batchTimeout) + } + prevMsgType = msgType + switch v := msg.(type) { + case *message.WriteMigrateTable: + err = migrateTable.append(ctx, v) + case *message.WriteInsert: + err = insert.append(ctx, v) + case *message.WriteDeleteStale: + err = deleteStale.append(ctx, v) + case *message.WriteDeleteRecord: + err = deleteRecord.append(ctx, v) + default: + panic("unknown message type") + } + if err != nil { + return err + } + case <-ticker.Chan(): + if err := flush(prevMsgType); err != nil { + return err + } + prevMsgType = writers.MsgTypeUnset + } + } + return flush(prevMsgType) +} + +// generic batch manager for most message types +type batchManager[A ~[]T, T message.WriteMessage] struct { + batch []T + writeFunc func(ctx context.Context, messages A) error +} + +func (m *batchManager[A, T]) append(ctx context.Context, msg T) error { + if len(m.batch) == cap(m.batch) { + if err := m.flush(ctx); err != nil { + return err + } + } + m.batch = append(m.batch, msg) + return nil +} + +func (m *batchManager[A, T]) flush(ctx context.Context) error { + if len(m.batch) == 0 { + return nil + } + + err := m.writeFunc(ctx, m.batch) + if err != nil { + return err + } + clear(m.batch) // GC can work + m.batch = m.batch[:0] + return nil +} + +// special batch manager for insert messages that also keeps track of the total size of the batch +type insertBatchManager struct { + batch message.WriteInserts + writeFunc func(ctx context.Context, messages message.WriteInserts) error + limit *batch.Cap + logger zerolog.Logger +} + +func (m *insertBatchManager) append(ctx context.Context, msg *message.WriteInsert) error { + add, toFlush, rest := batch.SliceRecord(msg.Record, m.limit) + if add != nil { + m.batch = append(m.batch, &message.WriteInsert{Record: add.Record}) + m.limit.AddSlice(add) + } + if len(toFlush) > 0 || rest != nil || m.limit.ReachedLimit() { + // flush current batch + if err := m.flush(ctx); err != nil { + return err + } + } + for _, sliceToFlush := range toFlush { + m.batch = append(m.batch, &message.WriteInsert{Record: sliceToFlush}) + m.limit.AddRows(sliceToFlush.NumRows()) + if err := m.flush(ctx); err != nil { + return err + } + } + + // set the remainder + if rest != nil { + m.batch = append(m.batch, &message.WriteInsert{Record: rest.Record}) + m.limit.AddSlice(rest) + } + + return nil +} + +func (m *insertBatchManager) flush(ctx context.Context) error { + rows := m.limit.Rows() + if rows == 0 { + // no rows to insert + return nil + } + start := time.Now() + err := m.writeFunc(ctx, m.batch) + duration := time.Since(start) + if err != nil { + m.logger.Err(err).Int64("len", rows).Dur("duration", duration).Msg("failed to write batch") + return err + } + m.logger.Debug().Int64("len", rows).Dur("duration", duration).Msg("batch written successfully") + + clear(m.batch) // GC can work + m.batch = m.batch[:0] + m.limit.Reset() + return nil +} diff --git a/plugin-sdk/writers/mixedbatchwriter/mixedbatchwriter_test.go b/plugin-sdk/writers/mixedbatchwriter/mixedbatchwriter_test.go new file mode 100644 index 00000000..9d9339c4 --- /dev/null +++ b/plugin-sdk/writers/mixedbatchwriter/mixedbatchwriter_test.go @@ -0,0 +1,326 @@ +package mixedbatchwriter + +import ( + "context" + "testing" + "time" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "golang.org/x/sync/errgroup" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/writers" +) + +type testMixedBatchClient struct { + receivedBatches [][]message.WriteMessage +} + +func (c *testMixedBatchClient) MigrateTableBatch(_ context.Context, messages message.WriteMigrateTables) error { + m := make([]message.WriteMessage, len(messages)) + for i, msg := range messages { + m[i] = msg + } + c.receivedBatches = append(c.receivedBatches, m) + return nil +} + +func (c *testMixedBatchClient) InsertBatch(_ context.Context, messages message.WriteInserts) error { + m := make([]message.WriteMessage, len(messages)) + for i, msg := range messages { + m[i] = msg + } + c.receivedBatches = append(c.receivedBatches, m) + return nil +} + +func (c *testMixedBatchClient) DeleteStaleBatch(_ context.Context, messages message.WriteDeleteStales) error { + m := make([]message.WriteMessage, len(messages)) + for i, msg := range messages { + m[i] = msg + } + c.receivedBatches = append(c.receivedBatches, m) + return nil +} + +func (c *testMixedBatchClient) DeleteRecordsBatch(_ context.Context, messages message.WriteDeleteRecords) error { + m := make([]message.WriteMessage, len(messages)) + for i, msg := range messages { + m[i] = msg + } + c.receivedBatches = append(c.receivedBatches, m) + return nil +} + +var _ Client = (*testMixedBatchClient)(nil) + +type testMessages struct { + migrateTable1 *message.WriteMigrateTable + migrateTable2 *message.WriteMigrateTable + insert1 *message.WriteInsert + insert2 *message.WriteInsert + deleteStale1 *message.WriteDeleteStale + deleteStale2 *message.WriteDeleteStale +} + +func getTestMessages() testMessages { + // message to create table1 + table1 := &schema.Table{ + Name: "table1", + Columns: []schema.Column{ + { + Name: "id", + Type: arrow.PrimitiveTypes.Int64, + }, + }, + } + msgMigrateTable1 := &message.WriteMigrateTable{ + Table: table1, + } + + // message to create table2 + table2 := &schema.Table{ + Name: "table2", + Columns: []schema.Column{ + { + Name: "id", + Type: arrow.PrimitiveTypes.Int64, + }, + }, + } + msgMigrateTable2 := &message.WriteMigrateTable{ + Table: table2, + } + + // message to insert into table1 + bldr1 := array.NewRecordBuilder(memory.DefaultAllocator, table1.ToArrowSchema()) + bldr1.Field(0).(*array.Int64Builder).Append(1) + rec1 := bldr1.NewRecord() + msgInsertTable1 := &message.WriteInsert{ + Record: rec1, + } + + // message to insert into table2 + bldr2 := array.NewRecordBuilder(memory.DefaultAllocator, table1.ToArrowSchema()) + bldr2.Field(0).(*array.Int64Builder).Append(1) + rec2 := bldr2.NewRecord() + msgInsertTable2 := &message.WriteInsert{ + Record: rec2, + } + + // message to delete stale from table1 + msgDeleteStale1 := &message.WriteDeleteStale{ + TableName: table1.Name, + SourceName: "my-source", + SyncTime: time.Now(), + } + msgDeleteStale2 := &message.WriteDeleteStale{ + TableName: table1.Name, + SourceName: "my-source", + SyncTime: time.Now(), + } + + return testMessages{ + migrateTable1: msgMigrateTable1, + migrateTable2: msgMigrateTable2, + insert1: msgInsertTable1, + insert2: msgInsertTable2, + deleteStale1: msgDeleteStale1, + deleteStale2: msgDeleteStale2, + } +} + +func TestMixedBatchWriter(t *testing.T) { + tm := getTestMessages() + testCases := []struct { + name string + messages []message.WriteMessage + wantBatches [][]message.WriteMessage + }{ + { + name: "create table, insert, delete stale", + messages: []message.WriteMessage{ + tm.migrateTable1, + tm.migrateTable2, + tm.insert1, + tm.insert2, + tm.deleteStale1, + tm.deleteStale2, + }, + wantBatches: [][]message.WriteMessage{ + {tm.migrateTable1, tm.migrateTable2}, + {tm.insert1, tm.insert2}, + {tm.deleteStale1, tm.deleteStale2}, + }, + }, + { + name: "interleaved messages", + messages: []message.WriteMessage{ + tm.migrateTable1, + tm.insert1, + tm.deleteStale1, + tm.migrateTable2, + tm.insert2, + tm.deleteStale2, + }, + wantBatches: [][]message.WriteMessage{ + {tm.migrateTable1}, + {tm.insert1}, + {tm.deleteStale1}, + {tm.migrateTable2}, + {tm.insert2}, + {tm.deleteStale2}, + }, + }, + { + name: "interleaved messages", + messages: []message.WriteMessage{ + tm.migrateTable1, + tm.migrateTable2, + tm.insert1, + tm.deleteStale2, + tm.insert2, + tm.deleteStale1, + }, + wantBatches: [][]message.WriteMessage{ + {tm.migrateTable1, tm.migrateTable2}, + {tm.insert1}, + {tm.deleteStale2}, + {tm.insert2}, + {tm.deleteStale1}, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + client := &testMixedBatchClient{ + receivedBatches: make([][]message.WriteMessage, 0), + } + wr, err := New(client) + if err != nil { + t.Fatal(err) + } + ch := make(chan message.WriteMessage, len(tc.messages)) + for _, msg := range tc.messages { + ch <- msg + } + close(ch) + if err := wr.Write(ctx, ch); err != nil { + t.Fatal(err) + } + if len(client.receivedBatches) != len(tc.wantBatches) { + t.Fatalf("got %d batches, want %d", len(client.receivedBatches), len(tc.wantBatches)) + } + for i, wantBatch := range tc.wantBatches { + if len(client.receivedBatches[i]) != len(wantBatch) { + t.Fatalf("got %d messages in batch %d, want %d", len(client.receivedBatches[i]), i, len(wantBatch)) + } + } + }) + } +} + +type mockTicker struct { + C chan time.Time + trigger chan struct{} +} + +func (m *mockTicker) Chan() <-chan time.Time { + return m.C +} + +func (m *mockTicker) Trigger() chan<- struct{} { + return m.trigger +} + +func (m *mockTicker) Stop() { + close(m.C) +} + +func (*mockTicker) Reset(_ time.Duration) {} + +func newMockTicker(trigger chan struct{}) *mockTicker { + c := make(chan time.Time) + go func() { + for range trigger { + c <- time.Now() + } + }() + return &mockTicker{ + C: c, + trigger: trigger, + } +} + +func TestMixedBatchWriterTimeout(t *testing.T) { + tm := getTestMessages() + cases := []struct { + name string + messages []message.WriteMessage + wantBatches [][]message.WriteMessage + }{ + { + name: "one_message_batches", + messages: []message.WriteMessage{ + tm.insert1, + tm.insert2, + }, + wantBatches: [][]message.WriteMessage{ + {tm.insert1}, + {tm.insert2}, + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + client := &testMixedBatchClient{ + receivedBatches: make([][]message.WriteMessage, 0), + } + triggerTimeout := make(chan struct{}) + defer close(triggerTimeout) + wr, err := New(client, + WithBatchSize(1000), + WithBatchSizeBytes(1000000), + withTickerFn(func(_ time.Duration) writers.Ticker { + return newMockTicker(triggerTimeout) + }), + ) + if err != nil { + t.Fatal(err) + } + ch := make(chan message.WriteMessage) + + eg := errgroup.Group{} + eg.Go(func() error { + return wr.Write(ctx, ch) + }) + + for _, msg := range tc.messages { + ch <- msg + time.Sleep(100 * time.Millisecond) + triggerTimeout <- struct{}{} + time.Sleep(100 * time.Millisecond) + } + close(ch) + err = eg.Wait() + if err != nil { + t.Fatalf("got error %v, want nil", err) + } + + if len(client.receivedBatches) != len(tc.wantBatches) { + t.Fatalf("got %d batches, want %d", len(client.receivedBatches), len(tc.wantBatches)) + } + for i, wantBatch := range tc.wantBatches { + if len(client.receivedBatches[i]) != len(wantBatch) { + t.Fatalf("got %d messages in batch %d, want %d", len(client.receivedBatches[i]), i, len(wantBatch)) + } + } + }) + } +} diff --git a/plugin-sdk/writers/mixedbatchwriter/unimplemented.go b/plugin-sdk/writers/mixedbatchwriter/unimplemented.go new file mode 100644 index 00000000..74c3269d --- /dev/null +++ b/plugin-sdk/writers/mixedbatchwriter/unimplemented.go @@ -0,0 +1,27 @@ +package mixedbatchwriter + +import ( + "context" + "fmt" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/plugin" +) + +type IgnoreMigrateTableBatch struct{} + +func (IgnoreMigrateTableBatch) MigrateTableBatch(context.Context, message.WriteMigrateTables) error { + return nil +} + +type UnimplementedDeleteStaleBatch struct{} + +func (UnimplementedDeleteStaleBatch) DeleteStaleBatch(context.Context, message.WriteDeleteStales) error { + return fmt.Errorf("DeleteStaleBatch: %w", plugin.ErrNotImplemented) +} + +type UnimplementedDeleteRecordsBatch struct{} + +func (UnimplementedDeleteRecordsBatch) DeleteRecordsBatch(context.Context, message.WriteDeleteRecords) error { + return fmt.Errorf("DeleteRecordsBatch: %w", plugin.ErrNotImplemented) +} diff --git a/plugin-sdk/writers/mixedbatchwriter/unimplemented_test.go b/plugin-sdk/writers/mixedbatchwriter/unimplemented_test.go new file mode 100644 index 00000000..521ea03e --- /dev/null +++ b/plugin-sdk/writers/mixedbatchwriter/unimplemented_test.go @@ -0,0 +1,20 @@ +package mixedbatchwriter_test + +import ( + "context" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/writers/mixedbatchwriter" +) + +type testDummyClient struct { + mixedbatchwriter.IgnoreMigrateTableBatch + mixedbatchwriter.UnimplementedDeleteStaleBatch + mixedbatchwriter.UnimplementedDeleteRecordsBatch +} + +func (testDummyClient) InsertBatch(context.Context, message.WriteInserts) error { + return nil +} + +var _ mixedbatchwriter.Client = (*testDummyClient)(nil) diff --git a/plugin-sdk/writers/msgtype.go b/plugin-sdk/writers/msgtype.go new file mode 100644 index 00000000..5e1b9480 --- /dev/null +++ b/plugin-sdk/writers/msgtype.go @@ -0,0 +1,31 @@ +package writers + +import ( + "reflect" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" +) + +type MsgType int + +const ( + MsgTypeUnset MsgType = iota + MsgTypeMigrateTable + MsgTypeInsert + MsgTypeDeleteStale + MsgTypeDeleteRecord +) + +func MsgID(msg message.WriteMessage) MsgType { + switch msg.(type) { + case *message.WriteMigrateTable: + return MsgTypeMigrateTable + case *message.WriteInsert: + return MsgTypeInsert + case *message.WriteDeleteStale: + return MsgTypeDeleteStale + case *message.WriteDeleteRecord: + return MsgTypeDeleteRecord + } + panic("unknown message type: " + reflect.TypeOf(msg).Name()) +} diff --git a/plugin-sdk/writers/streamingbatchwriter/mocktimer_test.go b/plugin-sdk/writers/streamingbatchwriter/mocktimer_test.go new file mode 100644 index 00000000..680e5485 --- /dev/null +++ b/plugin-sdk/writers/streamingbatchwriter/mocktimer_test.go @@ -0,0 +1,39 @@ +package streamingbatchwriter + +import ( + "sync" + "time" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/writers" +) + +type mockTicker struct { + expire chan time.Time + stopped sync.Once +} + +func (t *mockTicker) Stop() { + t.stopped.Do(func() { + close(t.expire) + }) +} + +func (t *mockTicker) Tick() { + t.expire <- time.Now() +} + +func (*mockTicker) Reset(time.Duration) {} + +func (t *mockTicker) Chan() <-chan time.Time { + return t.expire +} + +func newMockTicker() (writers.TickerFunc, func()) { + expire := make(chan time.Time) + t := &mockTicker{ + expire: expire, + } + return func(time.Duration) writers.Ticker { + return t + }, t.Tick +} diff --git a/plugin-sdk/writers/streamingbatchwriter/streamingbatchwriter.go b/plugin-sdk/writers/streamingbatchwriter/streamingbatchwriter.go new file mode 100644 index 00000000..659dfef1 --- /dev/null +++ b/plugin-sdk/writers/streamingbatchwriter/streamingbatchwriter.go @@ -0,0 +1,450 @@ +// Package streamingbatchwriter provides a writers.Writer implementation that writes to a client that implements the streamingbatchwriter.Client interface. +// +// Write messages are sent to the client with three separate methods: MigrateTable, WriteTable, and DeleteStale. Each method is called separate goroutines. +// Message types are processed in blocks: Receipt of a new message type will cause the previous message type processing to end (if it exists) which is signalled +// to the handler by closing the channel. The handler should return after processing all messages. +// +// For Insert messages (handled by WriteTable) each table creates separate goroutine. Number of goroutines is limited by the number of tables. +// Thus, each WriteTable invocation is for a single table (all messages sent to WriteTable are guaranteed to be for the same table). +// +// After a 'batch' is complete, the channel is closed. The handler is expected to block until the channel is closed and to keep processing in a streaming fashion. +// +// Batches are considered complete when: +// 1. The batch timeout is reached +// 2. The batch size is reached +// 3. The batch size in bytes is reached +// 4. A different message type is received +// +// Each handler can get invoked multiple times as new batches are processed. +// Handlers get invoked only if there's a message of that type at hand: First message of the batch is immediately available in the channel. +package streamingbatchwriter + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/rs/zerolog" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/internal/batch" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/writers" +) + +// Client is the interface that must be implemented by the client of StreamingBatchWriter. +type Client interface { + // MigrateTable should block and handle WriteMigrateTable messages until the channel is closed. + MigrateTable(context.Context, <-chan *message.WriteMigrateTable) error + + // DeleteStale should block and handle WriteDeleteStale messages until the channel is closed. + DeleteStale(context.Context, <-chan *message.WriteDeleteStale) error + + // DeleteRecords should block and handle WriteDeleteRecord messages until the channel is closed. + DeleteRecords(context.Context, <-chan *message.WriteDeleteRecord) error + + // WriteTable should block and handle writes to a single table until the channel is closed. Table metadata can be found in the first WriteInsert message. + // The channel is closed when all inserts in the batch have been sent. New batches, if any, will be sent on a new call to WriteTable. + WriteTable(context.Context, <-chan *message.WriteInsert) error +} + +type StreamingBatchWriter struct { + client Client + + insertWorkers map[string]*streamingWorkerManager[*message.WriteInsert] + migrateWorker *streamingWorkerManager[*message.WriteMigrateTable] + deleteStaleWorker *streamingWorkerManager[*message.WriteDeleteStale] + deleteRecordWorker *streamingWorkerManager[*message.WriteDeleteRecord] + + workersLock sync.RWMutex + workersWaitGroup sync.WaitGroup + + lastMsgType writers.MsgType + + logger zerolog.Logger + batchTimeout time.Duration + batchSizeRows int64 + batchSizeBytes int64 + + tickerFn writers.TickerFunc +} + +// Assert at compile-time that StreamingBatchWriter implements the Writer interface +var _ writers.Writer = (*StreamingBatchWriter)(nil) + +type Option func(*StreamingBatchWriter) + +func WithLogger(logger zerolog.Logger) Option { + return func(p *StreamingBatchWriter) { + p.logger = logger + } +} + +func WithBatchTimeout(timeout time.Duration) Option { + return func(p *StreamingBatchWriter) { + p.batchTimeout = timeout + } +} + +func WithBatchSizeRows(size int64) Option { + return func(p *StreamingBatchWriter) { + p.batchSizeRows = size + } +} + +func WithBatchSizeBytes(size int64) Option { + return func(p *StreamingBatchWriter) { + p.batchSizeBytes = size + } +} + +func withTickerFn(tickerFn writers.TickerFunc) Option { + return func(p *StreamingBatchWriter) { + p.tickerFn = tickerFn + } +} + +const ( + defaultBatchTimeoutSeconds = 20 + defaultBatchSize = 10000 + defaultBatchSizeBytes = 5 * 1024 * 1024 // 5 MiB +) + +func New(client Client, opts ...Option) (*StreamingBatchWriter, error) { + c := &StreamingBatchWriter{ + client: client, + insertWorkers: make(map[string]*streamingWorkerManager[*message.WriteInsert]), + logger: zerolog.Nop(), + batchTimeout: defaultBatchTimeoutSeconds * time.Second, + batchSizeRows: defaultBatchSize, + batchSizeBytes: defaultBatchSizeBytes, + tickerFn: writers.NewTicker, + } + for _, opt := range opts { + opt(c) + } + return c, nil +} + +func (w *StreamingBatchWriter) Flush(_ context.Context) error { + w.workersLock.RLock() + if w.migrateWorker != nil { + done := make(chan bool) + w.migrateWorker.flush <- done + <-done + } + if w.deleteStaleWorker != nil { + done := make(chan bool) + w.deleteStaleWorker.flush <- done + <-done + } + if w.deleteRecordWorker != nil { + done := make(chan bool) + w.deleteRecordWorker.flush <- done + <-done + } + for _, worker := range w.insertWorkers { + done := make(chan bool) + worker.flush <- done + <-done + } + w.workersLock.RUnlock() + return nil +} + +func (w *StreamingBatchWriter) Close(context.Context) error { + w.workersLock.Lock() + defer w.workersLock.Unlock() + for _, w := range w.insertWorkers { + close(w.ch) + } + if w.migrateWorker != nil { + close(w.migrateWorker.ch) + } + if w.deleteStaleWorker != nil { + close(w.deleteStaleWorker.ch) + } + if w.deleteRecordWorker != nil { + close(w.deleteRecordWorker.ch) + } + w.workersWaitGroup.Wait() + + w.insertWorkers = make(map[string]*streamingWorkerManager[*message.WriteInsert]) + w.migrateWorker = nil + w.deleteStaleWorker = nil + w.deleteRecordWorker = nil + w.lastMsgType = writers.MsgTypeUnset + + return nil +} + +func (w *StreamingBatchWriter) Write(ctx context.Context, msgs <-chan message.WriteMessage) error { + errCh := make(chan error) + defer close(errCh) + + go func() { + for err := range errCh { + w.logger.Err(err).Msg("error from StreamingBatchWriter") + } + }() + + for msg := range msgs { + msgType := writers.MsgID(msg) + if w.lastMsgType != writers.MsgTypeUnset && w.lastMsgType != msgType { + if err := w.Flush(ctx); err != nil { + return err + } + } + w.lastMsgType = msgType + if err := w.startWorker(ctx, errCh, msg); err != nil { + return err + } + } + + return w.Close(ctx) +} + +func (w *StreamingBatchWriter) startWorker(ctx context.Context, errCh chan<- error, msg message.WriteMessage) error { + var tableName string + + if mi, ok := msg.(*message.WriteInsert); ok { + md := mi.Record.Schema().Metadata() + tableName, ok = md.GetValue(schema.MetadataTableName) + if !ok { + return fmt.Errorf("table name not found in metadata") + } + } else { + tableName = msg.GetTable().Name + } + + switch m := msg.(type) { + case *message.WriteMigrateTable: + w.workersLock.Lock() + defer w.workersLock.Unlock() + if w.migrateWorker != nil { + w.migrateWorker.ch <- m + return nil + } + ch := make(chan *message.WriteMigrateTable) + w.migrateWorker = &streamingWorkerManager[*message.WriteMigrateTable]{ + ch: ch, + writeFunc: w.client.MigrateTable, + + flush: make(chan chan bool), + errCh: errCh, + + limit: batch.CappedAt(0, w.batchSizeRows), + batchTimeout: w.batchTimeout, + tickerFn: w.tickerFn, + } + + w.workersWaitGroup.Add(1) + go w.migrateWorker.run(ctx, &w.workersWaitGroup, tableName) + w.migrateWorker.ch <- m + return nil + case *message.WriteDeleteStale: + w.workersLock.Lock() + defer w.workersLock.Unlock() + if w.deleteStaleWorker != nil { + w.deleteStaleWorker.ch <- m + return nil + } + ch := make(chan *message.WriteDeleteStale) + w.deleteStaleWorker = &streamingWorkerManager[*message.WriteDeleteStale]{ + ch: ch, + writeFunc: w.client.DeleteStale, + + flush: make(chan chan bool), + errCh: errCh, + + limit: batch.CappedAt(0, w.batchSizeRows), + batchTimeout: w.batchTimeout, + tickerFn: w.tickerFn, + } + + w.workersWaitGroup.Add(1) + go w.deleteStaleWorker.run(ctx, &w.workersWaitGroup, tableName) + w.deleteStaleWorker.ch <- m + return nil + case *message.WriteInsert: + w.workersLock.RLock() + wr, ok := w.insertWorkers[tableName] + w.workersLock.RUnlock() + if ok { + wr.ch <- m + return nil + } + + ch := make(chan *message.WriteInsert) + wr = &streamingWorkerManager[*message.WriteInsert]{ + ch: ch, + writeFunc: w.client.WriteTable, + + flush: make(chan chan bool), + errCh: errCh, + + limit: batch.CappedAt(w.batchSizeBytes, w.batchSizeRows), + batchTimeout: w.batchTimeout, + tickerFn: w.tickerFn, + } + w.workersLock.Lock() + wrOld, ok := w.insertWorkers[tableName] + if ok { + w.workersLock.Unlock() + // some other goroutine could have already added the worker + // just send the message to it & discard our allocated worker + wrOld.ch <- m + return nil + } + w.insertWorkers[tableName] = wr + w.workersLock.Unlock() + + w.workersWaitGroup.Add(1) + go wr.run(ctx, &w.workersWaitGroup, tableName) + ch <- m + return nil + case *message.WriteDeleteRecord: + w.workersLock.Lock() + defer w.workersLock.Unlock() + if w.deleteRecordWorker != nil { + w.deleteRecordWorker.ch <- m + return nil + } + ch := make(chan *message.WriteDeleteRecord) + // TODO: flush all workers for nested tables as well (See https://github.com/cloudquery/plugin-sdk/issues/1296) + w.deleteRecordWorker = &streamingWorkerManager[*message.WriteDeleteRecord]{ + ch: ch, + writeFunc: w.client.DeleteRecords, + + flush: make(chan chan bool), + errCh: errCh, + + limit: batch.CappedAt(w.batchSizeBytes, w.batchSizeRows), + batchTimeout: w.batchTimeout, + tickerFn: w.tickerFn, + } + + w.workersWaitGroup.Add(1) + go w.deleteRecordWorker.run(ctx, &w.workersWaitGroup, tableName) + w.deleteRecordWorker.ch <- m + return nil + default: + return fmt.Errorf("unhandled message type: %T", msg) + } +} + +type streamingWorkerManager[T message.WriteMessage] struct { + ch chan T + writeFunc func(context.Context, <-chan T) error + + flush chan chan bool + errCh chan<- error + + limit *batch.Cap + batchTimeout time.Duration + tickerFn writers.TickerFunc +} + +func (s *streamingWorkerManager[T]) run(ctx context.Context, wg *sync.WaitGroup, tableName string) { + defer wg.Done() + var ( + clientCh chan T + clientErrCh chan error + open bool + ) + + ensureOpened := func() { + if open { + return + } + + clientCh = make(chan T) + clientErrCh = make(chan error, 1) + go func() { + defer close(clientErrCh) + defer func() { + if err := recover(); err != nil { + clientErrCh <- fmt.Errorf("panic: %v", err) + } + }() + clientErrCh <- s.writeFunc(ctx, clientCh) + }() + open = true + } + closeFlush := func() { + if open { + close(clientCh) + if err := <-clientErrCh; err != nil { + s.errCh <- fmt.Errorf("handler failed on %s: %w", tableName, err) + } + s.limit.Reset() + } + open = false + } + defer closeFlush() + + ticker := s.tickerFn(s.batchTimeout) + defer ticker.Stop() + + tickerCh, ctxDone := ticker.Chan(), ctx.Done() + + for { + select { + case r, ok := <-s.ch: + if !ok { + return + } + + if ins, ok := any(r).(*message.WriteInsert); ok { + add, toFlush, rest := batch.SliceRecord(ins.Record, s.limit) + if add != nil { + ensureOpened() + s.limit.AddSlice(add) + clientCh <- any(&message.WriteInsert{Record: add.Record}).(T) + } + if len(toFlush) > 0 || rest != nil || s.limit.ReachedLimit() { + // flush current batch + closeFlush() + ticker.Reset(s.batchTimeout) + } + for _, sliceToFlush := range toFlush { + ensureOpened() + s.limit.AddRows(sliceToFlush.NumRows()) + clientCh <- any(&message.WriteInsert{Record: sliceToFlush}).(T) + closeFlush() + ticker.Reset(s.batchTimeout) + } + + // set the remainder + if rest != nil { + ensureOpened() + s.limit.AddSlice(rest) + clientCh <- any(&message.WriteInsert{Record: rest.Record}).(T) + } + } else { + ensureOpened() + clientCh <- r + s.limit.AddRows(1) + if s.limit.ReachedLimit() { + closeFlush() + ticker.Reset(s.batchTimeout) + } + } + + case <-tickerCh: + if s.limit.Rows() > 0 { + closeFlush() + } + case done := <-s.flush: + if s.limit.Rows() > 0 { + closeFlush() + ticker.Reset(s.batchTimeout) + } + done <- true + case <-ctxDone: + // this means the request was cancelled + return // after this NO other call will succeed + } + } +} diff --git a/plugin-sdk/writers/streamingbatchwriter/streamingbatchwriter_test.go b/plugin-sdk/writers/streamingbatchwriter/streamingbatchwriter_test.go new file mode 100644 index 00000000..2a9ae1ef --- /dev/null +++ b/plugin-sdk/writers/streamingbatchwriter/streamingbatchwriter_test.go @@ -0,0 +1,415 @@ +package streamingbatchwriter + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" +) + +type messageType int + +const ( + messageTypeMigrateTable messageType = iota + messageTypeInsert + messageTypeDeleteStale + messageTypeDeleteRecord +) + +type testStreamingBatchClient struct { + mutex sync.Mutex + + inflight map[messageType]int + committed map[messageType]int + open map[messageType][]string +} + +func newClient() *testStreamingBatchClient { + return &testStreamingBatchClient{ + inflight: make(map[messageType]int), + committed: make(map[messageType]int), + open: make(map[messageType][]string), + } +} + +func (c *testStreamingBatchClient) MessageLen(t messageType) int { + c.mutex.Lock() + defer c.mutex.Unlock() + return c.committed[t] +} + +func (c *testStreamingBatchClient) InflightLen(t messageType) int { + c.mutex.Lock() + defer c.mutex.Unlock() + return c.inflight[t] +} + +func (c *testStreamingBatchClient) OpenLen(t messageType) int { + c.mutex.Lock() + defer c.mutex.Unlock() + return len(c.open[t]) +} + +func (c *testStreamingBatchClient) MigrateTable(ctx context.Context, msgs <-chan *message.WriteMigrateTable) error { + key := "" + for m := range msgs { + key = c.handleTypeMessage(ctx, messageTypeMigrateTable, m, key) + } + return c.handleTypeCommit(ctx, messageTypeMigrateTable, key) +} + +func (c *testStreamingBatchClient) WriteTable(ctx context.Context, msgs <-chan *message.WriteInsert) error { + key := "" + for m := range msgs { + key = c.handleTypeMessage(ctx, messageTypeInsert, m, key) + } + return c.handleTypeCommit(ctx, messageTypeInsert, key) +} + +func (c *testStreamingBatchClient) DeleteStale(ctx context.Context, msgs <-chan *message.WriteDeleteStale) error { + key := "" + for m := range msgs { + key = c.handleTypeMessage(ctx, messageTypeDeleteStale, m, key) + } + return c.handleTypeCommit(ctx, messageTypeDeleteStale, key) +} + +func (c *testStreamingBatchClient) DeleteRecords(ctx context.Context, msgs <-chan *message.WriteDeleteRecord) error { + key := "" + for m := range msgs { + key = c.handleTypeMessage(ctx, messageTypeDeleteRecord, m, key) + } + return c.handleTypeCommit(ctx, messageTypeDeleteRecord, key) +} + +func (c *testStreamingBatchClient) handleTypeMessage(_ context.Context, t messageType, msg message.WriteMessage, key string) string { + c.mutex.Lock() + defer c.mutex.Unlock() + + if key == "" { + key = msg.GetTable().Name + c.open[t] = append(c.open[t], key) + } + c.inflight[t]++ + + return key +} + +func (c *testStreamingBatchClient) handleTypeCommit(_ context.Context, t messageType, key string) error { + c.mutex.Lock() + defer c.mutex.Unlock() + + c.committed[t] += c.inflight[t] + c.inflight[t] = 0 + + for i, openTable := range c.open[t] { + if openTable == key { + c.open[t] = append(c.open[t][:i], c.open[t][i+1:]...) + break + } + } + + return nil +} + +var _ Client = (*testStreamingBatchClient)(nil) + +var streamingBatchTestTable = &schema.Table{ + Name: "table1", + Columns: []schema.Column{ + { + Name: "id", + Type: arrow.PrimitiveTypes.Int64, + }, + }, +} + +// TestBatchStreamFlushDifferentMessages tests that if writer receives a message of a new type all other pending batches are flushed. +func TestBatchStreamFlushDifferentMessages(t *testing.T) { + t.Parallel() + ctx := context.Background() + ch := make(chan message.WriteMessage) + + testClient := newClient() + wr, err := New(testClient) + if err != nil { + t.Fatal(err) + } + + errCh := make(chan error) + go func() { + errCh <- wr.Write(ctx, ch) + }() + + ch <- &message.WriteMigrateTable{Table: streamingBatchTestTable} + time.Sleep(50 * time.Millisecond) + + bldr := array.NewRecordBuilder(memory.DefaultAllocator, streamingBatchTestTable.ToArrowSchema()) + bldr.Field(0).(*array.Int64Builder).Append(1) + record := bldr.NewRecord() + + if l := testClient.MessageLen(messageTypeMigrateTable); l != 0 { + t.Fatalf("expected 0 migrate table messages, got %d", l) + } + + ch <- &message.WriteInsert{Record: record} + + waitForLength(t, testClient.MessageLen, messageTypeMigrateTable, 1) + + if l := testClient.MessageLen(messageTypeInsert); l != 0 { + t.Fatalf("expected 0 insert messages, got %d", l) + } + + ch <- &message.WriteMigrateTable{Table: streamingBatchTestTable} + + waitForLength(t, testClient.MessageLen, messageTypeInsert, 1) + + close(ch) + if err := <-errCh; err != nil { + t.Fatal(err) + } + + if l := testClient.OpenLen(messageTypeInsert); l != 0 { + t.Fatalf("expected 0 open tables, got %d", l) + } +} + +func TestStreamingBatchSizeRows(t *testing.T) { + t.Parallel() + ctx := context.Background() + ch := make(chan message.WriteMessage) + + testClient := newClient() + wr, err := New(testClient, WithBatchSizeRows(2)) + if err != nil { + t.Fatal(err) + } + + errCh := make(chan error) + go func() { + errCh <- wr.Write(ctx, ch) + }() + + table := schema.Table{Name: "table1", Columns: []schema.Column{{Name: "id", Type: arrow.PrimitiveTypes.Int64}}} + record := getRecord(table.ToArrowSchema(), 1) + ch <- &message.WriteInsert{ + Record: record, + } + time.Sleep(50 * time.Millisecond) + + if l := testClient.MessageLen(messageTypeInsert); l != 0 { + t.Fatalf("expected 0 insert messages, got %d", l) + } + + ch <- &message.WriteInsert{ + Record: record, + } + ch <- &message.WriteInsert{ // third message, because we flush before exceeding the limit and then save the third one + Record: record, + } + + waitForLength(t, testClient.MessageLen, messageTypeInsert, 2) + + close(ch) + if err := <-errCh; err != nil { + t.Fatal(err) + } + + if l := testClient.OpenLen(messageTypeInsert); l != 0 { + t.Fatalf("expected 0 open tables, got %d", l) + } + + if l := testClient.MessageLen(messageTypeInsert); l != 3 { + t.Fatalf("expected 3 insert messages, got %d", l) + } +} + +func TestStreamingBatchTimeout(t *testing.T) { + t.Parallel() + ctx := context.Background() + ch := make(chan message.WriteMessage) + + testClient := newClient() + tickerFn, tickFn := newMockTicker() + + wr, err := New(testClient, withTickerFn(tickerFn)) + if err != nil { + t.Fatal(err) + } + + errCh := make(chan error) + go func() { + errCh <- wr.Write(ctx, ch) + }() + + table := schema.Table{Name: "table1", Columns: []schema.Column{{Name: "id", Type: arrow.PrimitiveTypes.Int64}}} + record := getRecord(table.ToArrowSchema(), 1) + ch <- &message.WriteInsert{ + Record: record, + } + time.Sleep(50 * time.Millisecond) + + if l := testClient.MessageLen(messageTypeInsert); l != 0 { + t.Fatalf("expected 0 insert messages, got %d", l) + } + + // we need to wait for the batch to be flushed + time.Sleep(time.Millisecond * 50) + + if l := testClient.MessageLen(messageTypeInsert); l != 0 { + t.Fatalf("expected 0 insert messages, got %d", l) + } + + // flush + tickFn() + waitForLength(t, testClient.MessageLen, messageTypeInsert, 1) + + close(ch) + if err := <-errCh; err != nil { + t.Fatal(err) + } + + if l := testClient.OpenLen(messageTypeInsert); l != 0 { + t.Fatalf("expected 0 open tables, got %d", l) + } +} + +func TestStreamingBatchNoTimeout(t *testing.T) { + t.Parallel() + ctx := context.Background() + ch := make(chan message.WriteMessage) + + testClient := newClient() + wr, err := New(testClient, WithBatchTimeout(0), WithBatchSizeRows(2)) + if err != nil { + t.Fatal(err) + } + + errCh := make(chan error) + go func() { + errCh <- wr.Write(ctx, ch) + }() + + table := schema.Table{Name: "table1", Columns: []schema.Column{{Name: "id", Type: arrow.PrimitiveTypes.Int64}}} + record := getRecord(table.ToArrowSchema(), 1) + ch <- &message.WriteInsert{ + Record: record, + } + time.Sleep(50 * time.Millisecond) + + if l := testClient.MessageLen(messageTypeInsert); l != 0 { + t.Fatalf("expected 0 insert messages, got %d", l) + } + + time.Sleep(2 * time.Second) + + if l := testClient.MessageLen(messageTypeInsert); l != 0 { + t.Fatalf("expected 0 insert messages, got %d", l) + } + + ch <- &message.WriteInsert{ + Record: record, + } + ch <- &message.WriteInsert{ + Record: record, + } + + waitForLength(t, testClient.MessageLen, messageTypeInsert, 2) + + close(ch) + if err := <-errCh; err != nil { + t.Fatal(err) + } + + if l := testClient.OpenLen(messageTypeInsert); l != 0 { + t.Fatalf("expected 0 open tables, got %d", l) + } + + if l := testClient.MessageLen(messageTypeInsert); l != 3 { + t.Fatalf("expected 3 insert messages, got %d", l) + } +} + +func TestStreamingBatchUpserts(t *testing.T) { + t.Parallel() + ctx := context.Background() + ch := make(chan message.WriteMessage) + + testClient := newClient() + tickerFn, tickFn := newMockTicker() + wr, err := New(testClient, WithBatchSizeRows(2), withTickerFn(tickerFn)) + if err != nil { + t.Fatal(err) + } + + errCh := make(chan error) + go func() { + errCh <- wr.Write(ctx, ch) + }() + + table := schema.Table{Name: "table1", Columns: []schema.Column{{Name: "id", Type: arrow.PrimitiveTypes.Int64, PrimaryKey: true}}} + + bldr := array.NewRecordBuilder(memory.DefaultAllocator, table.ToArrowSchema()) + bldr.Field(0).(*array.Int64Builder).Append(1) + record := bldr.NewRecord() + + ch <- &message.WriteInsert{ + Record: record, + } + time.Sleep(50 * time.Millisecond) + + waitForLength(t, testClient.InflightLen, messageTypeInsert, 1) + waitForLength(t, testClient.MessageLen, messageTypeInsert, 0) + + ch <- &message.WriteInsert{ + Record: record, + } + time.Sleep(50 * time.Millisecond) + + // flush the batch + tickFn() + waitForLength(t, testClient.MessageLen, messageTypeInsert, 2) + + close(ch) + if err := <-errCh; err != nil { + t.Fatal(err) + } + + if l := testClient.OpenLen(messageTypeInsert); l != 0 { + t.Fatalf("expected 0 open tables, got %d", l) + } +} + +func waitForLength(t *testing.T, checkLen func(messageType) int, msgType messageType, want int) { + t.Helper() + lastValue := -1 + timeout := time.After(5 * time.Second) + for { + select { + case <-timeout: + t.Fatalf("timed out waiting for %v message length %d (last value: %d)", msgType, want, lastValue) + default: + if lastValue = checkLen(msgType); lastValue == want { + return + } + } + } +} + +func getRecord(sc *arrow.Schema, rows int) arrow.Record { + builder := array.NewRecordBuilder(memory.DefaultAllocator, sc) + defer builder.Release() + + for _, f := range builder.Fields() { + f.AppendEmptyValues(rows) + } + + return builder.NewRecord() +} diff --git a/plugin-sdk/writers/streamingbatchwriter/unimplemented.go b/plugin-sdk/writers/streamingbatchwriter/unimplemented.go new file mode 100644 index 00000000..18ad9208 --- /dev/null +++ b/plugin-sdk/writers/streamingbatchwriter/unimplemented.go @@ -0,0 +1,38 @@ +package streamingbatchwriter + +import ( + "context" + "fmt" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/plugin" +) + +// IgnoreMigrateTable is a dummy handler to consume WriteMigrateTable messages +type IgnoreMigrateTable struct{} + +func (IgnoreMigrateTable) MigrateTable(_ context.Context, ch <-chan *message.WriteMigrateTable) error { + // nolint:revive + for range ch { + } + return nil +} + +// UnimplementedDeleteStale is a dummy handler to consume and error on DeleteStale messages +type UnimplementedDeleteStale struct{} + +func (UnimplementedDeleteStale) DeleteStale(_ context.Context, ch <-chan *message.WriteDeleteStale) error { + // nolint:revive + for range ch { + } + return fmt.Errorf("DeleteStale: %w", plugin.ErrNotImplemented) +} + +type UnimplementedDeleteRecords struct{} + +func (UnimplementedDeleteRecords) DeleteRecords(_ context.Context, ch <-chan *message.WriteDeleteRecord) error { + // nolint:revive + for range ch { + } + return fmt.Errorf("DeleteRecords: %w", plugin.ErrNotImplemented) +} diff --git a/plugin-sdk/writers/streamingbatchwriter/unimplemented_test.go b/plugin-sdk/writers/streamingbatchwriter/unimplemented_test.go new file mode 100644 index 00000000..ad333934 --- /dev/null +++ b/plugin-sdk/writers/streamingbatchwriter/unimplemented_test.go @@ -0,0 +1,20 @@ +package streamingbatchwriter_test + +import ( + "context" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/writers/streamingbatchwriter" +) + +type testDummyClient struct { + streamingbatchwriter.IgnoreMigrateTable + streamingbatchwriter.UnimplementedDeleteStale + streamingbatchwriter.UnimplementedDeleteRecords +} + +func (testDummyClient) WriteTable(context.Context, <-chan *message.WriteInsert) error { + return nil +} + +var _ streamingbatchwriter.Client = (*testDummyClient)(nil) diff --git a/plugin-sdk/writers/ticker.go b/plugin-sdk/writers/ticker.go new file mode 100644 index 00000000..e6f1107a --- /dev/null +++ b/plugin-sdk/writers/ticker.go @@ -0,0 +1,38 @@ +package writers + +import ( + "time" +) + +type TickerFunc func(time.Duration) Ticker + +type Ticker interface { + Stop() + Reset(d time.Duration) + Chan() <-chan time.Time +} + +func NewTicker(interval time.Duration) Ticker { + if interval <= 0 { + return nopTicker{} + } + return &ticker{time.NewTicker(interval)} +} + +type ticker struct { + *time.Ticker +} + +func (t *ticker) Chan() <-chan time.Time { + return t.C +} + +type nopTicker struct{} + +func (nopTicker) Stop() {} + +func (nopTicker) Reset(_ time.Duration) {} + +func (nopTicker) Chan() <-chan time.Time { + return nil +} diff --git a/plugin-sdk/writers/writers.go b/plugin-sdk/writers/writers.go new file mode 100644 index 00000000..24d7aa3f --- /dev/null +++ b/plugin-sdk/writers/writers.go @@ -0,0 +1,11 @@ +package writers + +import ( + "context" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" +) + +type Writer interface { + Write(ctx context.Context, res <-chan message.WriteMessage) error +} diff --git a/plugin-sdk/writers/writers_test.go b/plugin-sdk/writers/writers_test.go new file mode 100644 index 00000000..0062437b --- /dev/null +++ b/plugin-sdk/writers/writers_test.go @@ -0,0 +1,219 @@ +package writers_test + +import ( + "context" + "math/rand" + "runtime" + "sort" + "strconv" + "testing" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "golang.org/x/exp/maps" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/writers" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/writers/batchwriter" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/writers/mixedbatchwriter" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/writers/streamingbatchwriter" +) + +type bCase struct { + name string + wr writers.Writer + rec func() arrow.Record +} + +func BenchmarkWriterMemory(b *testing.B) { + batchwriterOpts := map[string][]batchwriter.Option{ + "defaults": nil, + "batch10k bytes100M": {batchwriter.WithBatchSizeBytes(100000000), batchwriter.WithBatchSize(10000)}, + } + mixedbatchwriterOpts := map[string][]mixedbatchwriter.Option{ + "defaults": nil, + "batch10k bytes100M": {mixedbatchwriter.WithBatchSizeBytes(100000000), mixedbatchwriter.WithBatchSize(10000)}, + } + streamingbatchwriterOpts := map[string][]streamingbatchwriter.Option{ + "defaults": nil, + "bytes100M": {streamingbatchwriter.WithBatchSizeBytes(100000000)}, + } + + var bCases []bCase + bCases = append(bCases, writerMatrix("BatchWriter", batchwriter.New, newBatchWriterClient(), makeRecord, batchwriterOpts)...) + bCases = append(bCases, writerMatrix("BatchWriter wide", batchwriter.New, newBatchWriterClient(), makeWideRecord, batchwriterOpts)...) + bCases = append(bCases, writerMatrix("MixedBatchWriter", mixedbatchwriter.New, newMixedBatchWriterClient(), makeRecord, mixedbatchwriterOpts)...) + bCases = append(bCases, writerMatrix("MixedBatchWriter wide", mixedbatchwriter.New, newMixedBatchWriterClient(), makeWideRecord, mixedbatchwriterOpts)...) + bCases = append(bCases, writerMatrix("StreamingBatchWriter", streamingbatchwriter.New, newStreamingBatchWriterClient(), makeRecord, streamingbatchwriterOpts)...) + bCases = append(bCases, writerMatrix("StreamingBatchWriter wide", streamingbatchwriter.New, newStreamingBatchWriterClient(), makeWideRecord, streamingbatchwriterOpts)...) + + for _, c := range bCases { + c := c + b.Run(c.name, func(b *testing.B) { + var ( + mStart runtime.MemStats + mEnd runtime.MemStats + ) + + ch := make(chan message.WriteMessage) + errCh := make(chan error) + go func() { + defer close(errCh) + errCh <- c.wr.Write(context.Background(), ch) + }() + + runtime.ReadMemStats(&mStart) + b.ResetTimer() + for i := 0; i < b.N; i++ { + rec := c.rec() + ch <- &message.WriteInsert{ + Record: rec, + } + } + close(ch) + err := <-errCh + + b.StopTimer() + + if err != nil { + b.Fatal(err) + } + + runtime.ReadMemStats(&mEnd) + + allocatedBytes := mEnd.Alloc - mStart.Alloc + b.ReportMetric(float64(allocatedBytes)/float64(b.N), "bytes/op") // this is different from -benchmem result "B/op" + }) + } +} + +func makeRecord() func() arrow.Record { + table := &schema.Table{ + Name: "test_table", + Columns: schema.ColumnList{ + { + Name: "col1", + Type: arrow.BinaryTypes.String, + }, + }, + } + sc := table.ToArrowSchema() + + return func() arrow.Record { + bldr := array.NewRecordBuilder(memory.DefaultAllocator, sc) + bldr.Field(0).(*array.StringBuilder).Append("test") + return bldr.NewRecord() + } +} + +func makeWideRecord() func() arrow.Record { + table := &schema.Table{ + Name: "test_wide_table", + Columns: schema.ColumnList{ + { + Name: "col1", + Type: arrow.BinaryTypes.String, + }, + }, + } + + const numWideCols = 200 + randVals := make([]int64, numWideCols) + for i := 0; i < numWideCols; i++ { + table.Columns = append(table.Columns, schema.Column{ + Name: "wide_col" + strconv.Itoa(i), + Type: arrow.PrimitiveTypes.Int64, + }) + randVals[i] = rand.Int63() + } + sc := table.ToArrowSchema() + + return func() arrow.Record { + bldr := array.NewRecordBuilder(memory.DefaultAllocator, sc) + bldr.Field(0).(*array.StringBuilder).Append("test") + for i := 0; i < numWideCols; i++ { + bldr.Field(i + 1).(*array.Int64Builder).Append(randVals[i]) + } + return bldr.NewRecord() + } +} + +func writerMatrix[T writers.Writer, C any, O ~func(T)](prefix string, constructor func(C, ...O) (T, error), client C, recordMaker func() func() arrow.Record, optsMatrix map[string][]O) []bCase { + bCases := make([]bCase, 0, len(optsMatrix)) + + k := maps.Keys(optsMatrix) + sort.Strings(k) + + for _, name := range k { + opts := optsMatrix[name] + wr, err := constructor(client, opts...) + if err != nil { + panic(err) + } + bCases = append(bCases, bCase{ + name: prefix + " " + name, + wr: wr, + rec: recordMaker(), + }) + } + return bCases +} + +type mixedbatchwriterClient struct { + mixedbatchwriter.IgnoreMigrateTableBatch + mixedbatchwriter.UnimplementedDeleteStaleBatch + mixedbatchwriter.UnimplementedDeleteRecordsBatch +} + +func newMixedBatchWriterClient() mixedbatchwriter.Client { + return &mixedbatchwriterClient{} +} + +func (mixedbatchwriterClient) InsertBatch(_ context.Context, msgs message.WriteInserts) error { + for _, m := range msgs { + m.Record.Release() + } + return nil +} + +var _ mixedbatchwriter.Client = (*mixedbatchwriterClient)(nil) + +type batchwriterClient struct { + batchwriter.IgnoreMigrateTables + batchwriter.UnimplementedDeleteStale + batchwriter.UnimplementedDeleteRecord +} + +func newBatchWriterClient() batchwriter.Client { + return &batchwriterClient{} +} + +func (batchwriterClient) WriteTableBatch(_ context.Context, _ string, msgs message.WriteInserts) error { + for _, m := range msgs { + m.Record.Release() + } + return nil +} + +var _ batchwriter.Client = (*batchwriterClient)(nil) + +type streamingbatchwriterClient struct { + streamingbatchwriter.IgnoreMigrateTable + streamingbatchwriter.UnimplementedDeleteStale + streamingbatchwriter.UnimplementedDeleteRecords +} + +func newStreamingBatchWriterClient() streamingbatchwriter.Client { + return &streamingbatchwriterClient{} +} + +func (streamingbatchwriterClient) WriteTable(_ context.Context, ch <-chan *message.WriteInsert) error { + for m := range ch { + m.Record.Release() + } + return nil +} + +var _ streamingbatchwriter.Client = (*streamingbatchwriterClient)(nil) diff --git a/resources/plugin/client.go b/resources/plugin/client.go index 9411f230..30fb9cda 100644 --- a/resources/plugin/client.go +++ b/resources/plugin/client.go @@ -5,17 +5,18 @@ import ( "encoding/json" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/spec" "github.com/aws/aws-sdk-go-v2/aws" - "github.com/cloudquery/plugin-sdk/v4/message" - "github.com/cloudquery/plugin-sdk/v4/plugin" - "github.com/cloudquery/plugin-sdk/v4/scheduler" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/state" "github.com/rs/zerolog" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/spec" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/message" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/plugin" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/scheduler" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/state" ) const maxMsgSize = 100 * 1024 * 1024 // 100 MiB diff --git a/resources/plugin/plugin.go b/resources/plugin/plugin.go index a3f56220..7630e104 100644 --- a/resources/plugin/plugin.go +++ b/resources/plugin/plugin.go @@ -4,12 +4,13 @@ import ( "maps" "strings" - "github.com/OpsHelmInc/cloudquery/v2/client/spec" "github.com/aws/aws-sdk-go-v2/aws" - "github.com/cloudquery/plugin-sdk/v4/caser" - "github.com/cloudquery/plugin-sdk/v4/docs" - "github.com/cloudquery/plugin-sdk/v4/plugin" - "github.com/cloudquery/plugin-sdk/v4/schema" + + "github.com/OpsHelmInc/cloudquery/v2/client/spec" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/caser" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/docs" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/plugin" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" ) var ( diff --git a/resources/plugin/tables.go b/resources/plugin/tables.go index 8fc5c2e8..43fcd3bf 100644 --- a/resources/plugin/tables.go +++ b/resources/plugin/tables.go @@ -3,6 +3,9 @@ package plugin import ( "fmt" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" "github.com/OpsHelmInc/cloudquery/v2/resources/services/accessanalyzer" "github.com/OpsHelmInc/cloudquery/v2/resources/services/acm" "github.com/OpsHelmInc/cloudquery/v2/resources/services/acmpca" @@ -112,9 +115,6 @@ import ( "github.com/OpsHelmInc/cloudquery/v2/resources/services/wellarchitected" "github.com/OpsHelmInc/cloudquery/v2/resources/services/workspaces" "github.com/OpsHelmInc/cloudquery/v2/resources/services/xray" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - "github.com/cloudquery/plugin-sdk/v4/types" ) func GetTables() schema.Tables { diff --git a/resources/services/accessanalyzer/analyzer_archive_rules.go b/resources/services/accessanalyzer/analyzer_archive_rules.go index 26b76160..bf29038c 100644 --- a/resources/services/accessanalyzer/analyzer_archive_rules.go +++ b/resources/services/accessanalyzer/analyzer_archive_rules.go @@ -3,12 +3,13 @@ package accessanalyzer import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/accessanalyzer" "github.com/aws/aws-sdk-go-v2/service/accessanalyzer/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func analyzerArchiveRules() *schema.Table { diff --git a/resources/services/accessanalyzer/analyzer_findings.go b/resources/services/accessanalyzer/analyzer_findings.go index 87a501db..553ccff7 100644 --- a/resources/services/accessanalyzer/analyzer_findings.go +++ b/resources/services/accessanalyzer/analyzer_findings.go @@ -3,12 +3,13 @@ package accessanalyzer import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/accessanalyzer" "github.com/aws/aws-sdk-go-v2/service/accessanalyzer/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func analyzerFindings() *schema.Table { diff --git a/resources/services/accessanalyzer/analyzers.go b/resources/services/accessanalyzer/analyzers.go index 8a9b39e5..9f255d81 100644 --- a/resources/services/accessanalyzer/analyzers.go +++ b/resources/services/accessanalyzer/analyzers.go @@ -3,12 +3,13 @@ package accessanalyzer import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/accessanalyzer" "github.com/aws/aws-sdk-go-v2/service/accessanalyzer/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Analyzers() *schema.Table { diff --git a/resources/services/accessanalyzer/analyzers_mock_test.go b/resources/services/accessanalyzer/analyzers_mock_test.go index 8bf80183..15365ed6 100644 --- a/resources/services/accessanalyzer/analyzers_mock_test.go +++ b/resources/services/accessanalyzer/analyzers_mock_test.go @@ -3,13 +3,14 @@ package accessanalyzer import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/accessanalyzer" "github.com/aws/aws-sdk-go-v2/service/accessanalyzer/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildAccessAnalyzer(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/acm/certificates.go b/resources/services/acm/certificates.go index 07303c94..05613f26 100644 --- a/resources/services/acm/certificates.go +++ b/resources/services/acm/certificates.go @@ -3,13 +3,14 @@ package acm import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/acm" "github.com/aws/aws-sdk-go-v2/service/acm/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Certificates() *schema.Table { diff --git a/resources/services/acm/certificates_mock_test.go b/resources/services/acm/certificates_mock_test.go index 7872bf05..bcf0a5dd 100644 --- a/resources/services/acm/certificates_mock_test.go +++ b/resources/services/acm/certificates_mock_test.go @@ -3,14 +3,15 @@ package acm import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/acm" "github.com/aws/aws-sdk-go-v2/service/acm/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildACMCertificates(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/acmpca/certificate_authorities.go b/resources/services/acmpca/certificate_authorities.go index b66e0a99..22a763e2 100644 --- a/resources/services/acmpca/certificate_authorities.go +++ b/resources/services/acmpca/certificate_authorities.go @@ -3,13 +3,14 @@ package acmpca import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/acmpca" "github.com/aws/aws-sdk-go-v2/service/acmpca/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - cqtypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + cqtypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func CertificateAuthorities() *schema.Table { diff --git a/resources/services/acmpca/certificate_authorities_mock_test.go b/resources/services/acmpca/certificate_authorities_mock_test.go index 4a28f02a..5eb1bf10 100644 --- a/resources/services/acmpca/certificate_authorities_mock_test.go +++ b/resources/services/acmpca/certificate_authorities_mock_test.go @@ -3,14 +3,15 @@ package acmpca import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/acmpca" "github.com/aws/aws-sdk-go-v2/service/acmpca/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildACMPCACertificateAuthorities(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/amp/rule_groups_namespaces.go b/resources/services/amp/rule_groups_namespaces.go index d20bf387..d467038f 100644 --- a/resources/services/amp/rule_groups_namespaces.go +++ b/resources/services/amp/rule_groups_namespaces.go @@ -3,13 +3,14 @@ package amp import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/amp" "github.com/aws/aws-sdk-go-v2/service/amp/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ruleGroupsNamespaces() *schema.Table { diff --git a/resources/services/amp/rule_groups_namespaces_mock_test.go b/resources/services/amp/rule_groups_namespaces_mock_test.go index 37cf4bc8..4495d80e 100644 --- a/resources/services/amp/rule_groups_namespaces_mock_test.go +++ b/resources/services/amp/rule_groups_namespaces_mock_test.go @@ -3,12 +3,13 @@ package amp import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/amp" "github.com/aws/aws-sdk-go-v2/service/amp/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRuleGroupsNamespaces(t *testing.T, m *mocks.MockAmpClient) { diff --git a/resources/services/amp/workspaces.go b/resources/services/amp/workspaces.go index ff2dbcc7..de8106f7 100644 --- a/resources/services/amp/workspaces.go +++ b/resources/services/amp/workspaces.go @@ -3,14 +3,15 @@ package amp import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/amp" "github.com/aws/aws-sdk-go-v2/service/amp/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Workspaces() *schema.Table { diff --git a/resources/services/amp/workspaces_mock_test.go b/resources/services/amp/workspaces_mock_test.go index 7dc5c63a..45eac2a2 100644 --- a/resources/services/amp/workspaces_mock_test.go +++ b/resources/services/amp/workspaces_mock_test.go @@ -3,13 +3,14 @@ package amp import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/amp" "github.com/aws/aws-sdk-go-v2/service/amp/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildWorkspaces(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/amplify/apps.go b/resources/services/amplify/apps.go index e9d66c70..5c292f78 100644 --- a/resources/services/amplify/apps.go +++ b/resources/services/amplify/apps.go @@ -3,13 +3,14 @@ package amplify import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/amplify" "github.com/aws/aws-sdk-go-v2/service/amplify/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Apps() *schema.Table { diff --git a/resources/services/amplify/apps_mock_test.go b/resources/services/amplify/apps_mock_test.go index 77d9fb34..dbce7443 100644 --- a/resources/services/amplify/apps_mock_test.go +++ b/resources/services/amplify/apps_mock_test.go @@ -3,13 +3,14 @@ package amplify import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/amplify" "github.com/aws/aws-sdk-go-v2/service/amplify/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildApps(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/apigateway/api_keys.go b/resources/services/apigateway/api_keys.go index 24742926..db7a7312 100644 --- a/resources/services/apigateway/api_keys.go +++ b/resources/services/apigateway/api_keys.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigateway" "github.com/aws/aws-sdk-go-v2/service/apigateway/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ApiKeys() *schema.Table { @@ -55,6 +56,7 @@ func fetchApigatewayApiKeys(ctx context.Context, meta schema.ClientMeta, parent } return nil } + func resolveApigatewayAPIKeyArn(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { cl := meta.(*client.Client) ak := resource.Item.(types.ApiKey) diff --git a/resources/services/apigateway/api_keys_mock_test.go b/resources/services/apigateway/api_keys_mock_test.go index 3dae779f..1ec32bf3 100644 --- a/resources/services/apigateway/api_keys_mock_test.go +++ b/resources/services/apigateway/api_keys_mock_test.go @@ -3,13 +3,14 @@ package apigateway import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/apigateway" "github.com/aws/aws-sdk-go-v2/service/apigateway/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildApiKeysMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/apigateway/client_certificates.go b/resources/services/apigateway/client_certificates.go index 3ebfe9e1..88268339 100644 --- a/resources/services/apigateway/client_certificates.go +++ b/resources/services/apigateway/client_certificates.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigateway" "github.com/aws/aws-sdk-go-v2/service/apigateway/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ClientCertificates() *schema.Table { @@ -51,6 +52,7 @@ func fetchApigatewayClientCertificates(ctx context.Context, meta schema.ClientMe } return nil } + func resolveApigatewayClientCertificateArn(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { cl := meta.(*client.Client) cert := resource.Item.(types.ClientCertificate) diff --git a/resources/services/apigateway/client_certificates_mock_test.go b/resources/services/apigateway/client_certificates_mock_test.go index 70383797..ab8bc264 100644 --- a/resources/services/apigateway/client_certificates_mock_test.go +++ b/resources/services/apigateway/client_certificates_mock_test.go @@ -3,13 +3,14 @@ package apigateway import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/apigateway" "github.com/aws/aws-sdk-go-v2/service/apigateway/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildApigatewayClientCertificates(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/apigateway/domain_name_base_path_mappings.go b/resources/services/apigateway/domain_name_base_path_mappings.go index e53cf208..be80e78d 100644 --- a/resources/services/apigateway/domain_name_base_path_mappings.go +++ b/resources/services/apigateway/domain_name_base_path_mappings.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigateway" "github.com/aws/aws-sdk-go-v2/service/apigateway/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func domainNameBasePathMappings() *schema.Table { diff --git a/resources/services/apigateway/domain_names.go b/resources/services/apigateway/domain_names.go index 7779007a..82b1e911 100644 --- a/resources/services/apigateway/domain_names.go +++ b/resources/services/apigateway/domain_names.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigateway" "github.com/aws/aws-sdk-go-v2/service/apigateway/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func DomainNames() *schema.Table { @@ -54,6 +55,7 @@ func fetchApigatewayDomainNames(ctx context.Context, meta schema.ClientMeta, par } return nil } + func resolveApigatewayDomainNameArn(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { cl := meta.(*client.Client) domain := resource.Item.(types.DomainName) diff --git a/resources/services/apigateway/domain_names_mock_test.go b/resources/services/apigateway/domain_names_mock_test.go index 799b0152..ea3111a8 100644 --- a/resources/services/apigateway/domain_names_mock_test.go +++ b/resources/services/apigateway/domain_names_mock_test.go @@ -3,13 +3,14 @@ package apigateway import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/apigateway" "github.com/aws/aws-sdk-go-v2/service/apigateway/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildApigatewayDomainNames(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/apigateway/rest_api_authorizers.go b/resources/services/apigateway/rest_api_authorizers.go index 3883f311..56c90640 100644 --- a/resources/services/apigateway/rest_api_authorizers.go +++ b/resources/services/apigateway/rest_api_authorizers.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigateway" "github.com/aws/aws-sdk-go-v2/service/apigateway/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func restApiAuthorizers() *schema.Table { diff --git a/resources/services/apigateway/rest_api_deployments.go b/resources/services/apigateway/rest_api_deployments.go index 79b333a9..12b8d66a 100644 --- a/resources/services/apigateway/rest_api_deployments.go +++ b/resources/services/apigateway/rest_api_deployments.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigateway" "github.com/aws/aws-sdk-go-v2/service/apigateway/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func restApiDeployments() *schema.Table { diff --git a/resources/services/apigateway/rest_api_documentation_parts.go b/resources/services/apigateway/rest_api_documentation_parts.go index c46adab3..243cb889 100644 --- a/resources/services/apigateway/rest_api_documentation_parts.go +++ b/resources/services/apigateway/rest_api_documentation_parts.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigateway" "github.com/aws/aws-sdk-go-v2/service/apigateway/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func restApiDocumentationParts() *schema.Table { @@ -64,6 +65,7 @@ func fetchApigatewayRestApiDocumentationParts(ctx context.Context, meta schema.C } return nil } + func resolveApigatewayRestAPIDocumentationPartArn(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { cl := meta.(*client.Client) d := resource.Item.(types.DocumentationPart) diff --git a/resources/services/apigateway/rest_api_documentation_versions.go b/resources/services/apigateway/rest_api_documentation_versions.go index 6163c1a4..bb099362 100644 --- a/resources/services/apigateway/rest_api_documentation_versions.go +++ b/resources/services/apigateway/rest_api_documentation_versions.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigateway" "github.com/aws/aws-sdk-go-v2/service/apigateway/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func restApiDocumentationVersions() *schema.Table { @@ -64,6 +65,7 @@ func fetchApigatewayRestApiDocumentationVersions(ctx context.Context, meta schem } return nil } + func resolveApigatewayRestAPIDocumentationVersionArn(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { cl := meta.(*client.Client) v := resource.Item.(types.DocumentationVersion) diff --git a/resources/services/apigateway/rest_api_gateway_responses.go b/resources/services/apigateway/rest_api_gateway_responses.go index f070ae4d..50c0e881 100644 --- a/resources/services/apigateway/rest_api_gateway_responses.go +++ b/resources/services/apigateway/rest_api_gateway_responses.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigateway" "github.com/aws/aws-sdk-go-v2/service/apigateway/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func restApiGatewayResponses() *schema.Table { @@ -39,6 +40,7 @@ func restApiGatewayResponses() *schema.Table { }, } } + func fetchApigatewayRestApiGatewayResponses(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { r := parent.Item.(types.RestApi) cl := meta.(*client.Client) @@ -63,6 +65,7 @@ func fetchApigatewayRestApiGatewayResponses(ctx context.Context, meta schema.Cli } return nil } + func resolveApigatewayRestAPIGatewayResponseArn(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { cl := meta.(*client.Client) r := resource.Item.(types.GatewayResponse) diff --git a/resources/services/apigateway/rest_api_models.go b/resources/services/apigateway/rest_api_models.go index b528a868..a28c5f48 100644 --- a/resources/services/apigateway/rest_api_models.go +++ b/resources/services/apigateway/rest_api_models.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigateway" "github.com/aws/aws-sdk-go-v2/service/apigateway/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func restApiModels() *schema.Table { @@ -64,6 +65,7 @@ func fetchApigatewayRestApiModels(ctx context.Context, meta schema.ClientMeta, p } return nil } + func resolveApigatewayRestAPIModelArn(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { cl := meta.(*client.Client) m := resource.Item.(types.Model) diff --git a/resources/services/apigateway/rest_api_request_validators.go b/resources/services/apigateway/rest_api_request_validators.go index c647b186..91791055 100644 --- a/resources/services/apigateway/rest_api_request_validators.go +++ b/resources/services/apigateway/rest_api_request_validators.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigateway" "github.com/aws/aws-sdk-go-v2/service/apigateway/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func restApiRequestValidators() *schema.Table { @@ -64,6 +65,7 @@ func fetchApigatewayRestApiRequestValidators(ctx context.Context, meta schema.Cl } return nil } + func resolveApigatewayRestAPIRequestValidatorArn(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { cl := meta.(*client.Client) r := resource.Item.(types.RequestValidator) diff --git a/resources/services/apigateway/rest_api_resource_method_integration.go b/resources/services/apigateway/rest_api_resource_method_integration.go index 345e2b2c..40876e4a 100644 --- a/resources/services/apigateway/rest_api_resource_method_integration.go +++ b/resources/services/apigateway/rest_api_resource_method_integration.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigateway" "github.com/aws/aws-sdk-go-v2/service/apigateway/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func restApiResourceMethodIntegrations() *schema.Table { @@ -67,6 +68,7 @@ func fetchApigatewayRestApiResourceMethodIntegration(ctx context.Context, meta s res <- resp return nil } + func resolveApigatewayRestAPIResourceMethodIntegrationArn(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { cl := meta.(*client.Client) r := resource.Parent.Parent.Item.(types.Resource) diff --git a/resources/services/apigateway/rest_api_resource_methods.go b/resources/services/apigateway/rest_api_resource_methods.go index f9505d73..e27ef97c 100644 --- a/resources/services/apigateway/rest_api_resource_methods.go +++ b/resources/services/apigateway/rest_api_resource_methods.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigateway" "github.com/aws/aws-sdk-go-v2/service/apigateway/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func restApiResourceMethods() *schema.Table { diff --git a/resources/services/apigateway/rest_api_resources.go b/resources/services/apigateway/rest_api_resources.go index f201cc78..4a76428e 100644 --- a/resources/services/apigateway/rest_api_resources.go +++ b/resources/services/apigateway/rest_api_resources.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigateway" "github.com/aws/aws-sdk-go-v2/service/apigateway/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func restApiResources() *schema.Table { @@ -62,6 +63,7 @@ func fetchApigatewayRestApiResources(ctx context.Context, meta schema.ClientMeta } return nil } + func resolveApigatewayRestAPIResourceArn(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { cl := meta.(*client.Client) r := resource.Item.(types.Resource) diff --git a/resources/services/apigateway/rest_api_stages.go b/resources/services/apigateway/rest_api_stages.go index a242685d..e66893cc 100644 --- a/resources/services/apigateway/rest_api_stages.go +++ b/resources/services/apigateway/rest_api_stages.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigateway" "github.com/aws/aws-sdk-go-v2/service/apigateway/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func restApiStages() *schema.Table { @@ -59,6 +60,7 @@ func fetchApigatewayRestApiStages(ctx context.Context, meta schema.ClientMeta, p return nil } + func resolveApigatewayRestAPIStageArn(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { cl := meta.(*client.Client) s := resource.Item.(types.Stage) diff --git a/resources/services/apigateway/rest_apis.go b/resources/services/apigateway/rest_apis.go index f81f4fa1..d817c80b 100644 --- a/resources/services/apigateway/rest_apis.go +++ b/resources/services/apigateway/rest_apis.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigateway" "github.com/aws/aws-sdk-go-v2/service/apigateway/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func RestApis() *schema.Table { @@ -64,6 +65,7 @@ func fetchApigatewayRestApis(ctx context.Context, meta schema.ClientMeta, parent } return nil } + func resolveApigatewayRestAPIArn(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { cl := meta.(*client.Client) rapi := resource.Item.(types.RestApi) diff --git a/resources/services/apigateway/rest_apis_mock_test.go b/resources/services/apigateway/rest_apis_mock_test.go index a1c53e41..db989dd3 100644 --- a/resources/services/apigateway/rest_apis_mock_test.go +++ b/resources/services/apigateway/rest_apis_mock_test.go @@ -3,13 +3,14 @@ package apigateway import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/apigateway" "github.com/aws/aws-sdk-go-v2/service/apigateway/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildApigatewayRestApis(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/apigateway/usage_plan_keys.go b/resources/services/apigateway/usage_plan_keys.go index 0d018761..752251cf 100644 --- a/resources/services/apigateway/usage_plan_keys.go +++ b/resources/services/apigateway/usage_plan_keys.go @@ -1,11 +1,12 @@ package apigateway import ( - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/apigateway/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func usagePlanKeys() *schema.Table { diff --git a/resources/services/apigateway/usage_plans.go b/resources/services/apigateway/usage_plans.go index 2992b2cc..ccb664c9 100644 --- a/resources/services/apigateway/usage_plans.go +++ b/resources/services/apigateway/usage_plans.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigateway" "github.com/aws/aws-sdk-go-v2/service/apigateway/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func UsagePlans() *schema.Table { @@ -55,6 +56,7 @@ func fetchApigatewayUsagePlans(ctx context.Context, meta schema.ClientMeta, pare } return nil } + func resolveApigatewayUsagePlanArn(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { cl := meta.(*client.Client) up := resource.Item.(types.UsagePlan) @@ -66,6 +68,7 @@ func resolveApigatewayUsagePlanArn(ctx context.Context, meta schema.ClientMeta, Resource: fmt.Sprintf("/usageplans/%s", aws.ToString(up.Id)), }.String()) } + func fetchApigatewayUsagePlanKeys(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { r := parent.Item.(types.UsagePlan) cl := meta.(*client.Client) @@ -82,6 +85,7 @@ func fetchApigatewayUsagePlanKeys(ctx context.Context, meta schema.ClientMeta, p } return nil } + func resolveApigatewayUsagePlanKeyArn(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { cl := meta.(*client.Client) up := resource.Parent.Item.(types.UsagePlan) diff --git a/resources/services/apigateway/usage_plans_mock_test.go b/resources/services/apigateway/usage_plans_mock_test.go index ce30f63f..1b7ac7b2 100644 --- a/resources/services/apigateway/usage_plans_mock_test.go +++ b/resources/services/apigateway/usage_plans_mock_test.go @@ -3,13 +3,14 @@ package apigateway import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/apigateway" "github.com/aws/aws-sdk-go-v2/service/apigateway/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildApigatewayUsagePlans(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/apigateway/vpc_links.go b/resources/services/apigateway/vpc_links.go index 54c4fbd2..5c2a8199 100644 --- a/resources/services/apigateway/vpc_links.go +++ b/resources/services/apigateway/vpc_links.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigateway" "github.com/aws/aws-sdk-go-v2/service/apigateway/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func VpcLinks() *schema.Table { @@ -52,6 +53,7 @@ func fetchApigatewayVpcLinks(ctx context.Context, meta schema.ClientMeta, parent } return nil } + func resolveApigatewayVpcLinkArn(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { cl := meta.(*client.Client) link := resource.Item.(types.VpcLink) diff --git a/resources/services/apigateway/vpc_links_mock_test.go b/resources/services/apigateway/vpc_links_mock_test.go index 89c40252..e7c9b021 100644 --- a/resources/services/apigateway/vpc_links_mock_test.go +++ b/resources/services/apigateway/vpc_links_mock_test.go @@ -3,13 +3,14 @@ package apigateway import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/apigateway" "github.com/aws/aws-sdk-go-v2/service/apigateway/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildApigatewayVpcLinks(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/apigatewayv2/api_authorizers.go b/resources/services/apigatewayv2/api_authorizers.go index 2d8bb3ab..ab852d67 100644 --- a/resources/services/apigatewayv2/api_authorizers.go +++ b/resources/services/apigatewayv2/api_authorizers.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func apiAuthorizers() *schema.Table { diff --git a/resources/services/apigatewayv2/api_deployments.go b/resources/services/apigatewayv2/api_deployments.go index 49b43d65..da668f13 100644 --- a/resources/services/apigatewayv2/api_deployments.go +++ b/resources/services/apigatewayv2/api_deployments.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func apiDeployments() *schema.Table { @@ -44,6 +45,7 @@ func apiDeployments() *schema.Table { }, } } + func fetchApigatewayv2ApiDeployments(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { r := parent.Item.(types.Api) config := apigatewayv2.GetDeploymentsInput{ diff --git a/resources/services/apigatewayv2/api_integration_responses.go b/resources/services/apigatewayv2/api_integration_responses.go index 5becfc03..a2d291a0 100644 --- a/resources/services/apigatewayv2/api_integration_responses.go +++ b/resources/services/apigatewayv2/api_integration_responses.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func apiIntegrationResponses() *schema.Table { diff --git a/resources/services/apigatewayv2/api_integrations.go b/resources/services/apigatewayv2/api_integrations.go index 055d1f2e..1b475163 100644 --- a/resources/services/apigatewayv2/api_integrations.go +++ b/resources/services/apigatewayv2/api_integrations.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func apiIntegrations() *schema.Table { diff --git a/resources/services/apigatewayv2/api_models.go b/resources/services/apigatewayv2/api_models.go index 27a1f41d..10c20725 100644 --- a/resources/services/apigatewayv2/api_models.go +++ b/resources/services/apigatewayv2/api_models.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func apiModels() *schema.Table { @@ -49,6 +50,7 @@ func apiModels() *schema.Table { }, } } + func fetchApigatewayv2ApiModels(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { r := parent.Item.(types.Api) config := apigatewayv2.GetModelsInput{ diff --git a/resources/services/apigatewayv2/api_route_responses.go b/resources/services/apigatewayv2/api_route_responses.go index 610a0d50..c64d9297 100644 --- a/resources/services/apigatewayv2/api_route_responses.go +++ b/resources/services/apigatewayv2/api_route_responses.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func apiRouteResponses() *schema.Table { diff --git a/resources/services/apigatewayv2/api_routes.go b/resources/services/apigatewayv2/api_routes.go index b91a5c9c..f3b1e56e 100644 --- a/resources/services/apigatewayv2/api_routes.go +++ b/resources/services/apigatewayv2/api_routes.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func apiRoutes() *schema.Table { diff --git a/resources/services/apigatewayv2/api_stages.go b/resources/services/apigatewayv2/api_stages.go index 167a7a13..f78c698c 100644 --- a/resources/services/apigatewayv2/api_stages.go +++ b/resources/services/apigatewayv2/api_stages.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func apiStages() *schema.Table { diff --git a/resources/services/apigatewayv2/apis.go b/resources/services/apigatewayv2/apis.go index 114c0cf5..ed67f243 100644 --- a/resources/services/apigatewayv2/apis.go +++ b/resources/services/apigatewayv2/apis.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Apis() *schema.Table { diff --git a/resources/services/apigatewayv2/apis_mock_test.go b/resources/services/apigatewayv2/apis_mock_test.go index eb850076..7bab4dea 100644 --- a/resources/services/apigatewayv2/apis_mock_test.go +++ b/resources/services/apigatewayv2/apis_mock_test.go @@ -3,13 +3,14 @@ package apigatewayv2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildApigatewayv2Apis(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/apigatewayv2/domain_name_rest_api_mappings.go b/resources/services/apigatewayv2/domain_name_rest_api_mappings.go index 7ba176d0..0db6f5f8 100644 --- a/resources/services/apigatewayv2/domain_name_rest_api_mappings.go +++ b/resources/services/apigatewayv2/domain_name_rest_api_mappings.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func domainNameRestApiMappings() *schema.Table { diff --git a/resources/services/apigatewayv2/domain_names.go b/resources/services/apigatewayv2/domain_names.go index c7021e51..0d2a0f01 100644 --- a/resources/services/apigatewayv2/domain_names.go +++ b/resources/services/apigatewayv2/domain_names.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func DomainNames() *schema.Table { diff --git a/resources/services/apigatewayv2/domain_names_mock_test.go b/resources/services/apigatewayv2/domain_names_mock_test.go index 30d26786..d25150af 100644 --- a/resources/services/apigatewayv2/domain_names_mock_test.go +++ b/resources/services/apigatewayv2/domain_names_mock_test.go @@ -3,13 +3,14 @@ package apigatewayv2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildApigatewayv2DomainNames(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/apigatewayv2/vpc_links.go b/resources/services/apigatewayv2/vpc_links.go index b30e67ab..3dbdddb2 100644 --- a/resources/services/apigatewayv2/vpc_links.go +++ b/resources/services/apigatewayv2/vpc_links.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func VpcLinks() *schema.Table { diff --git a/resources/services/apigatewayv2/vpc_links_mock_test.go b/resources/services/apigatewayv2/vpc_links_mock_test.go index e8216d36..ddbf015d 100644 --- a/resources/services/apigatewayv2/vpc_links_mock_test.go +++ b/resources/services/apigatewayv2/vpc_links_mock_test.go @@ -3,13 +3,14 @@ package apigatewayv2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2" "github.com/aws/aws-sdk-go-v2/service/apigatewayv2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildApigatewayv2VpcLinks(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/appconfig/applications.go b/resources/services/appconfig/applications.go index 9ae4d074..6f62889d 100644 --- a/resources/services/appconfig/applications.go +++ b/resources/services/appconfig/applications.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/appconfig" "github.com/aws/aws-sdk-go-v2/service/appconfig/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Applications() *schema.Table { diff --git a/resources/services/appconfig/applications_mock_test.go b/resources/services/appconfig/applications_mock_test.go index cf375cc9..191e1da1 100644 --- a/resources/services/appconfig/applications_mock_test.go +++ b/resources/services/appconfig/applications_mock_test.go @@ -3,13 +3,14 @@ package appconfig import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/appconfig" "github.com/aws/aws-sdk-go-v2/service/appconfig/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildApps(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/appconfig/configuration_profiles.go b/resources/services/appconfig/configuration_profiles.go index d0bbd251..4242bdf0 100644 --- a/resources/services/appconfig/configuration_profiles.go +++ b/resources/services/appconfig/configuration_profiles.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/appconfig" "github.com/aws/aws-sdk-go-v2/service/appconfig/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func configurationProfiles() *schema.Table { diff --git a/resources/services/appconfig/deployment_strategies.go b/resources/services/appconfig/deployment_strategies.go index ac26fb1c..998c17ee 100644 --- a/resources/services/appconfig/deployment_strategies.go +++ b/resources/services/appconfig/deployment_strategies.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/appconfig" "github.com/aws/aws-sdk-go-v2/service/appconfig/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func DeploymentStrategies() *schema.Table { diff --git a/resources/services/appconfig/deployment_strategies_mock_test.go b/resources/services/appconfig/deployment_strategies_mock_test.go index eb1415b3..70b8cba5 100644 --- a/resources/services/appconfig/deployment_strategies_mock_test.go +++ b/resources/services/appconfig/deployment_strategies_mock_test.go @@ -3,13 +3,14 @@ package appconfig import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/appconfig" "github.com/aws/aws-sdk-go-v2/service/appconfig/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDeploymentStrategies(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/appconfig/environments.go b/resources/services/appconfig/environments.go index fe6848dd..286b212b 100644 --- a/resources/services/appconfig/environments.go +++ b/resources/services/appconfig/environments.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/appconfig" "github.com/aws/aws-sdk-go-v2/service/appconfig/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func environments() *schema.Table { diff --git a/resources/services/appconfig/hosted_config_versions.go b/resources/services/appconfig/hosted_config_versions.go index 908d2b0a..8993ef57 100644 --- a/resources/services/appconfig/hosted_config_versions.go +++ b/resources/services/appconfig/hosted_config_versions.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/appconfig" "github.com/aws/aws-sdk-go-v2/service/appconfig/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func hostedConfigurationVersions() *schema.Table { diff --git a/resources/services/appflow/flows.go b/resources/services/appflow/flows.go index 30751b4b..0055ca06 100644 --- a/resources/services/appflow/flows.go +++ b/resources/services/appflow/flows.go @@ -3,12 +3,13 @@ package appflow import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/appflow" "github.com/aws/aws-sdk-go-v2/service/appflow/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Flows() *schema.Table { diff --git a/resources/services/appflow/flows_mock_test.go b/resources/services/appflow/flows_mock_test.go index 5e8ea9c0..788d3165 100644 --- a/resources/services/appflow/flows_mock_test.go +++ b/resources/services/appflow/flows_mock_test.go @@ -3,13 +3,14 @@ package appflow import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/appflow" "github.com/aws/aws-sdk-go-v2/service/appflow/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildFlows(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/applicationautoscaling/policies.go b/resources/services/applicationautoscaling/policies.go index 0db4ecac..33856c55 100644 --- a/resources/services/applicationautoscaling/policies.go +++ b/resources/services/applicationautoscaling/policies.go @@ -3,12 +3,13 @@ package applicationautoscaling import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/applicationautoscaling" "github.com/aws/aws-sdk-go-v2/service/applicationautoscaling/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Policies() *schema.Table { diff --git a/resources/services/applicationautoscaling/policies_mock_test.go b/resources/services/applicationautoscaling/policies_mock_test.go index d0efc15f..8ca3af2d 100644 --- a/resources/services/applicationautoscaling/policies_mock_test.go +++ b/resources/services/applicationautoscaling/policies_mock_test.go @@ -3,13 +3,14 @@ package applicationautoscaling import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/applicationautoscaling" "github.com/aws/aws-sdk-go-v2/service/applicationautoscaling/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildApplicationAutoscalingPoliciesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/applicationautoscaling/scalable_targets.go b/resources/services/applicationautoscaling/scalable_targets.go index 2bd7c525..2573afa3 100644 --- a/resources/services/applicationautoscaling/scalable_targets.go +++ b/resources/services/applicationautoscaling/scalable_targets.go @@ -3,12 +3,13 @@ package applicationautoscaling import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/applicationautoscaling" "github.com/aws/aws-sdk-go-v2/service/applicationautoscaling/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ScalableTargets() *schema.Table { diff --git a/resources/services/applicationautoscaling/scalable_targets_mock_test.go b/resources/services/applicationautoscaling/scalable_targets_mock_test.go index cb447e9e..bbcdbdf2 100644 --- a/resources/services/applicationautoscaling/scalable_targets_mock_test.go +++ b/resources/services/applicationautoscaling/scalable_targets_mock_test.go @@ -3,13 +3,14 @@ package applicationautoscaling import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/applicationautoscaling" "github.com/aws/aws-sdk-go-v2/service/applicationautoscaling/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildScalableTargets(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/applicationautoscaling/scheduled_actions.go b/resources/services/applicationautoscaling/scheduled_actions.go index cd064584..0e82b60e 100644 --- a/resources/services/applicationautoscaling/scheduled_actions.go +++ b/resources/services/applicationautoscaling/scheduled_actions.go @@ -3,12 +3,13 @@ package applicationautoscaling import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/applicationautoscaling" "github.com/aws/aws-sdk-go-v2/service/applicationautoscaling/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ScheduledActions() *schema.Table { diff --git a/resources/services/applicationautoscaling/scheduled_actions_mock_test.go b/resources/services/applicationautoscaling/scheduled_actions_mock_test.go index 09c9a6e3..e6c76390 100644 --- a/resources/services/applicationautoscaling/scheduled_actions_mock_test.go +++ b/resources/services/applicationautoscaling/scheduled_actions_mock_test.go @@ -3,13 +3,14 @@ package applicationautoscaling import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/applicationautoscaling" "github.com/aws/aws-sdk-go-v2/service/applicationautoscaling/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildScheduledActions(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/appmesh/meshes.go b/resources/services/appmesh/meshes.go index d78e008c..6ff3b3bc 100644 --- a/resources/services/appmesh/meshes.go +++ b/resources/services/appmesh/meshes.go @@ -3,13 +3,14 @@ package appmesh import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/appmesh" "github.com/aws/aws-sdk-go-v2/service/appmesh/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Meshes() *schema.Table { diff --git a/resources/services/appmesh/meshes_mock_test.go b/resources/services/appmesh/meshes_mock_test.go index 36ebbded..98f64fd4 100644 --- a/resources/services/appmesh/meshes_mock_test.go +++ b/resources/services/appmesh/meshes_mock_test.go @@ -3,14 +3,15 @@ package appmesh import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/appmesh" "github.com/aws/aws-sdk-go-v2/service/appmesh/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildMeshes(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/appmesh/virtual_gateways.go b/resources/services/appmesh/virtual_gateways.go index 83f8b763..50885fe3 100644 --- a/resources/services/appmesh/virtual_gateways.go +++ b/resources/services/appmesh/virtual_gateways.go @@ -3,12 +3,13 @@ package appmesh import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/appmesh" "github.com/aws/aws-sdk-go-v2/service/appmesh/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func virtualGateways() *schema.Table { diff --git a/resources/services/appmesh/virtual_nodes.go b/resources/services/appmesh/virtual_nodes.go index e230d8c7..584d8c4a 100644 --- a/resources/services/appmesh/virtual_nodes.go +++ b/resources/services/appmesh/virtual_nodes.go @@ -3,12 +3,13 @@ package appmesh import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/appmesh" "github.com/aws/aws-sdk-go-v2/service/appmesh/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func virtualNodes() *schema.Table { diff --git a/resources/services/appmesh/virtual_routers.go b/resources/services/appmesh/virtual_routers.go index f542ecbb..db4a7e68 100644 --- a/resources/services/appmesh/virtual_routers.go +++ b/resources/services/appmesh/virtual_routers.go @@ -3,12 +3,13 @@ package appmesh import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/appmesh" "github.com/aws/aws-sdk-go-v2/service/appmesh/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func virtualRouters() *schema.Table { diff --git a/resources/services/appmesh/virtual_services.go b/resources/services/appmesh/virtual_services.go index d7ed7421..ef9f1cf2 100644 --- a/resources/services/appmesh/virtual_services.go +++ b/resources/services/appmesh/virtual_services.go @@ -3,12 +3,13 @@ package appmesh import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/appmesh" "github.com/aws/aws-sdk-go-v2/service/appmesh/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func virtualServices() *schema.Table { diff --git a/resources/services/apprunner/auto_scaling_configuration_mock_test.go b/resources/services/apprunner/auto_scaling_configuration_mock_test.go index 2876970d..f08ef9ae 100644 --- a/resources/services/apprunner/auto_scaling_configuration_mock_test.go +++ b/resources/services/apprunner/auto_scaling_configuration_mock_test.go @@ -3,13 +3,14 @@ package apprunner import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/apprunner" "github.com/aws/aws-sdk-go-v2/service/apprunner/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildApprunnerAutoScalingConfigurationsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/apprunner/auto_scaling_configurations.go b/resources/services/apprunner/auto_scaling_configurations.go index df2ff678..b8626616 100644 --- a/resources/services/apprunner/auto_scaling_configurations.go +++ b/resources/services/apprunner/auto_scaling_configurations.go @@ -3,13 +3,14 @@ package apprunner import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/apprunner" "github.com/aws/aws-sdk-go-v2/service/apprunner/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func AutoScalingConfigurations() *schema.Table { @@ -56,6 +57,7 @@ func fetchApprunnerAutoScalingConfigurations(ctx context.Context, meta schema.Cl } return nil } + func getAutoScalingConfiguration(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource) error { cl := meta.(*client.Client) svc := cl.Services(client.AWSServiceApprunner).Apprunner diff --git a/resources/services/apprunner/connections.go b/resources/services/apprunner/connections.go index 851eb48f..9f598d8b 100644 --- a/resources/services/apprunner/connections.go +++ b/resources/services/apprunner/connections.go @@ -3,13 +3,14 @@ package apprunner import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/apprunner" "github.com/aws/aws-sdk-go-v2/service/apprunner/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Connections() *schema.Table { diff --git a/resources/services/apprunner/connections_mock_test.go b/resources/services/apprunner/connections_mock_test.go index fdbab753..cd0d6fde 100644 --- a/resources/services/apprunner/connections_mock_test.go +++ b/resources/services/apprunner/connections_mock_test.go @@ -3,13 +3,14 @@ package apprunner import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/apprunner" "github.com/aws/aws-sdk-go-v2/service/apprunner/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildConnections(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/apprunner/custom_domains.go b/resources/services/apprunner/custom_domains.go index a291c75e..5acdc537 100644 --- a/resources/services/apprunner/custom_domains.go +++ b/resources/services/apprunner/custom_domains.go @@ -3,12 +3,13 @@ package apprunner import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/apprunner" "github.com/aws/aws-sdk-go-v2/service/apprunner/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func customDomains() *schema.Table { diff --git a/resources/services/apprunner/observability_configuration_mock_test.go b/resources/services/apprunner/observability_configuration_mock_test.go index 93668d30..888d5558 100644 --- a/resources/services/apprunner/observability_configuration_mock_test.go +++ b/resources/services/apprunner/observability_configuration_mock_test.go @@ -3,13 +3,14 @@ package apprunner import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/apprunner" "github.com/aws/aws-sdk-go-v2/service/apprunner/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildObservabilityConfiguration(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/apprunner/observability_configurations.go b/resources/services/apprunner/observability_configurations.go index 0d93c0ed..72321dd1 100644 --- a/resources/services/apprunner/observability_configurations.go +++ b/resources/services/apprunner/observability_configurations.go @@ -3,13 +3,14 @@ package apprunner import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/apprunner" "github.com/aws/aws-sdk-go-v2/service/apprunner/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func ObservabilityConfigurations() *schema.Table { diff --git a/resources/services/apprunner/operations.go b/resources/services/apprunner/operations.go index 06dfcadc..8d42c64e 100644 --- a/resources/services/apprunner/operations.go +++ b/resources/services/apprunner/operations.go @@ -3,12 +3,13 @@ package apprunner import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/apprunner" "github.com/aws/aws-sdk-go-v2/service/apprunner/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func operations() *schema.Table { diff --git a/resources/services/apprunner/services.go b/resources/services/apprunner/services.go index 3dae0b17..884bc429 100644 --- a/resources/services/apprunner/services.go +++ b/resources/services/apprunner/services.go @@ -3,13 +3,14 @@ package apprunner import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/apprunner" "github.com/aws/aws-sdk-go-v2/service/apprunner/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Services() *schema.Table { diff --git a/resources/services/apprunner/services_mock_test.go b/resources/services/apprunner/services_mock_test.go index 0ff2cff6..07b69313 100644 --- a/resources/services/apprunner/services_mock_test.go +++ b/resources/services/apprunner/services_mock_test.go @@ -3,13 +3,14 @@ package apprunner import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/apprunner" "github.com/aws/aws-sdk-go-v2/service/apprunner/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildApprunnerGraphqlApisMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/apprunner/tag_fetch.go b/resources/services/apprunner/tag_fetch.go index d732f5b9..1e18350f 100644 --- a/resources/services/apprunner/tag_fetch.go +++ b/resources/services/apprunner/tag_fetch.go @@ -4,11 +4,12 @@ import ( "context" "strings" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/apprunner" - "github.com/cloudquery/plugin-sdk/v4/schema" "github.com/thoas/go-funk" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" ) func resolveApprunnerTags(path string) schema.ColumnResolver { diff --git a/resources/services/apprunner/vpc_connector_mock_test.go b/resources/services/apprunner/vpc_connector_mock_test.go index 5907f6f1..0bc042de 100644 --- a/resources/services/apprunner/vpc_connector_mock_test.go +++ b/resources/services/apprunner/vpc_connector_mock_test.go @@ -3,13 +3,14 @@ package apprunner import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/apprunner" "github.com/aws/aws-sdk-go-v2/service/apprunner/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildApprunnerVpcConnectorsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/apprunner/vpc_connectors.go b/resources/services/apprunner/vpc_connectors.go index 85c468e3..fec2a805 100644 --- a/resources/services/apprunner/vpc_connectors.go +++ b/resources/services/apprunner/vpc_connectors.go @@ -3,13 +3,14 @@ package apprunner import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/apprunner" "github.com/aws/aws-sdk-go-v2/service/apprunner/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func VpcConnectors() *schema.Table { diff --git a/resources/services/apprunner/vpc_ingress_connection_mock_test.go b/resources/services/apprunner/vpc_ingress_connection_mock_test.go index 414a9b99..ab5a5561 100644 --- a/resources/services/apprunner/vpc_ingress_connection_mock_test.go +++ b/resources/services/apprunner/vpc_ingress_connection_mock_test.go @@ -3,13 +3,14 @@ package apprunner import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/apprunner" "github.com/aws/aws-sdk-go-v2/service/apprunner/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildApprunnerVpcIngressConnectionsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/apprunner/vpc_ingress_connections.go b/resources/services/apprunner/vpc_ingress_connections.go index adbdebfd..9b5262f1 100644 --- a/resources/services/apprunner/vpc_ingress_connections.go +++ b/resources/services/apprunner/vpc_ingress_connections.go @@ -3,13 +3,14 @@ package apprunner import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/apprunner" "github.com/aws/aws-sdk-go-v2/service/apprunner/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func VpcIngressConnections() *schema.Table { diff --git a/resources/services/appstream/app_blocks.go b/resources/services/appstream/app_blocks.go index aae5aeab..ede0de11 100644 --- a/resources/services/appstream/app_blocks.go +++ b/resources/services/appstream/app_blocks.go @@ -3,13 +3,14 @@ package appstream import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/appstream" "github.com/aws/aws-sdk-go-v2/service/appstream/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func AppBlocks() *schema.Table { diff --git a/resources/services/appstream/app_blocks_mock_test.go b/resources/services/appstream/app_blocks_mock_test.go index 8052b125..4ccb8c41 100644 --- a/resources/services/appstream/app_blocks_mock_test.go +++ b/resources/services/appstream/app_blocks_mock_test.go @@ -3,13 +3,14 @@ package appstream import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/appstream" "github.com/aws/aws-sdk-go-v2/service/appstream/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildAppstreamAppBlocksMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -30,6 +31,7 @@ func buildAppstreamAppBlocksMock(t *testing.T, ctrl *gomock.Controller) client.S Appstream: m, } } + func TestAppstreamAppBlocks(t *testing.T) { client.AwsMockTestHelper(t, AppBlocks(), buildAppstreamAppBlocksMock, client.TestOptions{}) } diff --git a/resources/services/appstream/application_fleet_associations.go b/resources/services/appstream/application_fleet_associations.go index 3f34aead..5966ec49 100644 --- a/resources/services/appstream/application_fleet_associations.go +++ b/resources/services/appstream/application_fleet_associations.go @@ -3,13 +3,14 @@ package appstream import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/appstream" "github.com/aws/aws-sdk-go-v2/service/appstream/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func applicationFleetAssociations() *schema.Table { diff --git a/resources/services/appstream/applications.go b/resources/services/appstream/applications.go index d38f856d..d2475d26 100644 --- a/resources/services/appstream/applications.go +++ b/resources/services/appstream/applications.go @@ -3,13 +3,14 @@ package appstream import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/appstream" "github.com/aws/aws-sdk-go-v2/service/appstream/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Applications() *schema.Table { diff --git a/resources/services/appstream/applications_mock_test.go b/resources/services/appstream/applications_mock_test.go index 198749df..d144b660 100644 --- a/resources/services/appstream/applications_mock_test.go +++ b/resources/services/appstream/applications_mock_test.go @@ -3,13 +3,14 @@ package appstream import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/appstream" "github.com/aws/aws-sdk-go-v2/service/appstream/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildAppstreamApplicationsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/appstream/fleets.go b/resources/services/appstream/fleets.go index ce132e43..caf9f4a6 100644 --- a/resources/services/appstream/fleets.go +++ b/resources/services/appstream/fleets.go @@ -3,13 +3,14 @@ package appstream import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/appstream" "github.com/aws/aws-sdk-go-v2/service/appstream/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Fleets() *schema.Table { diff --git a/resources/services/appstream/fleets_mock_test.go b/resources/services/appstream/fleets_mock_test.go index 116e8b74..7805da2d 100644 --- a/resources/services/appstream/fleets_mock_test.go +++ b/resources/services/appstream/fleets_mock_test.go @@ -3,13 +3,14 @@ package appstream import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/appstream" "github.com/aws/aws-sdk-go-v2/service/appstream/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildAppstreamFleetsMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -30,6 +31,7 @@ func buildAppstreamFleetsMock(t *testing.T, ctrl *gomock.Controller) client.Serv Appstream: m, } } + func TestAppstreamFleets(t *testing.T) { client.AwsMockTestHelper(t, Fleets(), buildAppstreamFleetsMock, client.TestOptions{}) } diff --git a/resources/services/appstream/image_builders.go b/resources/services/appstream/image_builders.go index 2b35e0cb..775e3d65 100644 --- a/resources/services/appstream/image_builders.go +++ b/resources/services/appstream/image_builders.go @@ -3,13 +3,14 @@ package appstream import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/appstream" "github.com/aws/aws-sdk-go-v2/service/appstream/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ImageBuilders() *schema.Table { diff --git a/resources/services/appstream/image_builders_mock_test.go b/resources/services/appstream/image_builders_mock_test.go index d2ff694a..d0439d1d 100644 --- a/resources/services/appstream/image_builders_mock_test.go +++ b/resources/services/appstream/image_builders_mock_test.go @@ -3,13 +3,14 @@ package appstream import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/appstream" "github.com/aws/aws-sdk-go-v2/service/appstream/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildAppstreamImageBuildersMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -30,6 +31,7 @@ func buildAppstreamImageBuildersMock(t *testing.T, ctrl *gomock.Controller) clie Appstream: m, } } + func TestAppstreamImageBuilders(t *testing.T) { client.AwsMockTestHelper(t, ImageBuilders(), buildAppstreamImageBuildersMock, client.TestOptions{}) } diff --git a/resources/services/appstream/images.go b/resources/services/appstream/images.go index 0a2eacba..6c9a950d 100644 --- a/resources/services/appstream/images.go +++ b/resources/services/appstream/images.go @@ -3,13 +3,14 @@ package appstream import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/appstream" "github.com/aws/aws-sdk-go-v2/service/appstream/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Images() *schema.Table { diff --git a/resources/services/appstream/images_mock_test.go b/resources/services/appstream/images_mock_test.go index 0327b54a..6b9db8c3 100644 --- a/resources/services/appstream/images_mock_test.go +++ b/resources/services/appstream/images_mock_test.go @@ -3,13 +3,14 @@ package appstream import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/appstream" "github.com/aws/aws-sdk-go-v2/service/appstream/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildAppstreamImagesMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -30,6 +31,7 @@ func buildAppstreamImagesMock(t *testing.T, ctrl *gomock.Controller) client.Serv Appstream: m, } } + func TestAppstreamImages(t *testing.T) { client.AwsMockTestHelper(t, Images(), buildAppstreamImagesMock, client.TestOptions{}) } diff --git a/resources/services/appstream/stack_entitlements.go b/resources/services/appstream/stack_entitlements.go index 3f795841..1c0aebcc 100644 --- a/resources/services/appstream/stack_entitlements.go +++ b/resources/services/appstream/stack_entitlements.go @@ -3,13 +3,14 @@ package appstream import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/appstream" "github.com/aws/aws-sdk-go-v2/service/appstream/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func stackEntitlements() *schema.Table { diff --git a/resources/services/appstream/stack_user_associations.go b/resources/services/appstream/stack_user_associations.go index 46506eea..2526d1ff 100644 --- a/resources/services/appstream/stack_user_associations.go +++ b/resources/services/appstream/stack_user_associations.go @@ -3,13 +3,14 @@ package appstream import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/appstream" "github.com/aws/aws-sdk-go-v2/service/appstream/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func stackUserAssociations() *schema.Table { diff --git a/resources/services/appstream/stacks.go b/resources/services/appstream/stacks.go index 7193ea29..21e83a55 100644 --- a/resources/services/appstream/stacks.go +++ b/resources/services/appstream/stacks.go @@ -3,13 +3,14 @@ package appstream import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/appstream" "github.com/aws/aws-sdk-go-v2/service/appstream/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Stacks() *schema.Table { diff --git a/resources/services/appstream/stacks_mock_test.go b/resources/services/appstream/stacks_mock_test.go index 7bf4b3d5..0ebe18c5 100644 --- a/resources/services/appstream/stacks_mock_test.go +++ b/resources/services/appstream/stacks_mock_test.go @@ -3,13 +3,14 @@ package appstream import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/appstream" "github.com/aws/aws-sdk-go-v2/service/appstream/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildAppstreamStacksMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/appstream/users.go b/resources/services/appstream/users.go index 9548c329..68ef4e03 100644 --- a/resources/services/appstream/users.go +++ b/resources/services/appstream/users.go @@ -3,13 +3,14 @@ package appstream import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/appstream" "github.com/aws/aws-sdk-go-v2/service/appstream/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Users() *schema.Table { diff --git a/resources/services/appstream/users_mock_test.go b/resources/services/appstream/users_mock_test.go index 209c7004..b0567175 100644 --- a/resources/services/appstream/users_mock_test.go +++ b/resources/services/appstream/users_mock_test.go @@ -3,13 +3,14 @@ package appstream import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/appstream" "github.com/aws/aws-sdk-go-v2/service/appstream/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildAppstreamUsersMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/appsync/graphql_apis.go b/resources/services/appsync/graphql_apis.go index b8b8c6b1..0b4b2bb2 100644 --- a/resources/services/appsync/graphql_apis.go +++ b/resources/services/appsync/graphql_apis.go @@ -3,13 +3,14 @@ package appsync import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/appsync" "github.com/aws/aws-sdk-go-v2/service/appsync/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func GraphqlApis() *schema.Table { diff --git a/resources/services/appsync/graphql_apis_mock_test.go b/resources/services/appsync/graphql_apis_mock_test.go index 443f4fcc..e7926869 100644 --- a/resources/services/appsync/graphql_apis_mock_test.go +++ b/resources/services/appsync/graphql_apis_mock_test.go @@ -3,13 +3,14 @@ package appsync import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/appsync" "github.com/aws/aws-sdk-go-v2/service/appsync/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildAppsyncGraphqlApisMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/athena/data_catalog_database_tables.go b/resources/services/athena/data_catalog_database_tables.go index bc3fb18c..af118986 100644 --- a/resources/services/athena/data_catalog_database_tables.go +++ b/resources/services/athena/data_catalog_database_tables.go @@ -3,12 +3,13 @@ package athena import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/athena" "github.com/aws/aws-sdk-go-v2/service/athena/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func dataCatalogDatabaseTables() *schema.Table { diff --git a/resources/services/athena/data_catalog_databases.go b/resources/services/athena/data_catalog_databases.go index 9f96a31f..bc31a5d9 100644 --- a/resources/services/athena/data_catalog_databases.go +++ b/resources/services/athena/data_catalog_databases.go @@ -3,12 +3,13 @@ package athena import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/athena" "github.com/aws/aws-sdk-go-v2/service/athena/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func dataCatalogDatabases() *schema.Table { diff --git a/resources/services/athena/data_catalogs.go b/resources/services/athena/data_catalogs.go index aa141b8e..a7731f5b 100644 --- a/resources/services/athena/data_catalogs.go +++ b/resources/services/athena/data_catalogs.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/athena" "github.com/aws/aws-sdk-go-v2/service/athena/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func DataCatalogs() *schema.Table { diff --git a/resources/services/athena/data_catalogs_mock_test.go b/resources/services/athena/data_catalogs_mock_test.go index 06347e58..e21599dd 100644 --- a/resources/services/athena/data_catalogs_mock_test.go +++ b/resources/services/athena/data_catalogs_mock_test.go @@ -3,12 +3,13 @@ package athena import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/athena" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDataCatalogs(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/athena/work_group_named_queries.go b/resources/services/athena/work_group_named_queries.go index 0212ad3c..64033d6f 100644 --- a/resources/services/athena/work_group_named_queries.go +++ b/resources/services/athena/work_group_named_queries.go @@ -3,13 +3,14 @@ package athena import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/athena" "github.com/aws/aws-sdk-go-v2/service/athena/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func workGroupNamedQueries() *schema.Table { diff --git a/resources/services/athena/work_group_prepared_statements.go b/resources/services/athena/work_group_prepared_statements.go index 7892bdc9..acfe1d70 100644 --- a/resources/services/athena/work_group_prepared_statements.go +++ b/resources/services/athena/work_group_prepared_statements.go @@ -3,12 +3,13 @@ package athena import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/athena" "github.com/aws/aws-sdk-go-v2/service/athena/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func workGroupPreparedStatements() *schema.Table { diff --git a/resources/services/athena/work_group_query_executions.go b/resources/services/athena/work_group_query_executions.go index 1bea8cb4..d62d768b 100644 --- a/resources/services/athena/work_group_query_executions.go +++ b/resources/services/athena/work_group_query_executions.go @@ -3,13 +3,14 @@ package athena import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/athena" "github.com/aws/aws-sdk-go-v2/service/athena/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func workGroupQueryExecutions() *schema.Table { diff --git a/resources/services/athena/work_groups.go b/resources/services/athena/work_groups.go index 399ff6d2..96f1ece6 100644 --- a/resources/services/athena/work_groups.go +++ b/resources/services/athena/work_groups.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/athena" "github.com/aws/aws-sdk-go-v2/service/athena/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func WorkGroups() *schema.Table { diff --git a/resources/services/athena/work_groups_mock_test.go b/resources/services/athena/work_groups_mock_test.go index 07a989b9..c11157a3 100644 --- a/resources/services/athena/work_groups_mock_test.go +++ b/resources/services/athena/work_groups_mock_test.go @@ -3,12 +3,13 @@ package athena import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/athena" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildWorkGroups(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/auditmanager/assesments.go b/resources/services/auditmanager/assesments.go index ae927c33..71c0e69e 100644 --- a/resources/services/auditmanager/assesments.go +++ b/resources/services/auditmanager/assesments.go @@ -3,12 +3,13 @@ package auditmanager import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/auditmanager" "github.com/aws/aws-sdk-go-v2/service/auditmanager/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Assessments() *schema.Table { diff --git a/resources/services/auditmanager/assesments_mock_test.go b/resources/services/auditmanager/assesments_mock_test.go index 2023474b..23e87739 100644 --- a/resources/services/auditmanager/assesments_mock_test.go +++ b/resources/services/auditmanager/assesments_mock_test.go @@ -3,13 +3,14 @@ package auditmanager import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/auditmanager" "github.com/aws/aws-sdk-go-v2/service/auditmanager/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildAssessments(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/autoscaling/group_lifecycle_hooks.go b/resources/services/autoscaling/group_lifecycle_hooks.go index 511c4c5c..9ae91a60 100644 --- a/resources/services/autoscaling/group_lifecycle_hooks.go +++ b/resources/services/autoscaling/group_lifecycle_hooks.go @@ -3,13 +3,14 @@ package autoscaling import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/autoscaling/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/autoscaling" "github.com/aws/aws-sdk-go-v2/service/autoscaling/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/autoscaling/models" ) func groupLifecycleHooks() *schema.Table { diff --git a/resources/services/autoscaling/group_scaling_policies.go b/resources/services/autoscaling/group_scaling_policies.go index e1a1b433..2160137f 100644 --- a/resources/services/autoscaling/group_scaling_policies.go +++ b/resources/services/autoscaling/group_scaling_policies.go @@ -3,13 +3,14 @@ package autoscaling import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/autoscaling/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/autoscaling" "github.com/aws/aws-sdk-go-v2/service/autoscaling/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/autoscaling/models" ) func groupScalingPolicies() *schema.Table { diff --git a/resources/services/autoscaling/groups.go b/resources/services/autoscaling/groups.go index b60243ae..d13e8ae9 100644 --- a/resources/services/autoscaling/groups.go +++ b/resources/services/autoscaling/groups.go @@ -3,15 +3,16 @@ package autoscaling import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/autoscaling/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/autoscaling" "github.com/aws/aws-sdk-go-v2/service/autoscaling/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/autoscaling/models" ) func Groups() *schema.Table { @@ -148,6 +149,7 @@ func resolveAutoscalingGroupLoadBalancers(ctx context.Context, meta schema.Clien } return resource.Set(c.Name, j) } + func resolveAutoscalingGroupLoadBalancerTargetGroups(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { p := resource.Item.(models.AutoScalingGroupWrapper) cl := meta.(*client.Client) diff --git a/resources/services/autoscaling/groups_mock_test.go b/resources/services/autoscaling/groups_mock_test.go index 3585536d..879ef7e6 100644 --- a/resources/services/autoscaling/groups_mock_test.go +++ b/resources/services/autoscaling/groups_mock_test.go @@ -3,12 +3,13 @@ package autoscaling import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/autoscaling" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildAutoscalingGroups(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/autoscaling/launch_configuration_mock_test.go b/resources/services/autoscaling/launch_configuration_mock_test.go index 937da439..23eacaad 100644 --- a/resources/services/autoscaling/launch_configuration_mock_test.go +++ b/resources/services/autoscaling/launch_configuration_mock_test.go @@ -3,13 +3,14 @@ package autoscaling import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/autoscaling" "github.com/aws/aws-sdk-go-v2/service/autoscaling/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildAutoscalingLaunchConfigurationsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/autoscaling/launch_configurations.go b/resources/services/autoscaling/launch_configurations.go index 6320e16c..e3492a55 100644 --- a/resources/services/autoscaling/launch_configurations.go +++ b/resources/services/autoscaling/launch_configurations.go @@ -3,12 +3,13 @@ package autoscaling import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/autoscaling" "github.com/aws/aws-sdk-go-v2/service/autoscaling/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func LaunchConfigurations() *schema.Table { diff --git a/resources/services/autoscaling/scheduled_actions.go b/resources/services/autoscaling/scheduled_actions.go index e637eed1..3ff2e43b 100644 --- a/resources/services/autoscaling/scheduled_actions.go +++ b/resources/services/autoscaling/scheduled_actions.go @@ -3,13 +3,14 @@ package autoscaling import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/autoscaling" "github.com/aws/aws-sdk-go-v2/service/autoscaling/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ScheduledActions() *schema.Table { diff --git a/resources/services/autoscaling/scheduled_actions_mock_test.go b/resources/services/autoscaling/scheduled_actions_mock_test.go index c91e46a3..4c3b1935 100644 --- a/resources/services/autoscaling/scheduled_actions_mock_test.go +++ b/resources/services/autoscaling/scheduled_actions_mock_test.go @@ -3,13 +3,14 @@ package autoscaling import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/autoscaling" "github.com/aws/aws-sdk-go-v2/service/autoscaling/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildAutoscalingSheduledActionMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/backup/plan_selections.go b/resources/services/backup/plan_selections.go index fe5fe8d5..474c430e 100644 --- a/resources/services/backup/plan_selections.go +++ b/resources/services/backup/plan_selections.go @@ -3,12 +3,13 @@ package backup import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/backup" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func planSelections() *schema.Table { diff --git a/resources/services/backup/plans.go b/resources/services/backup/plans.go index ae1323b5..4d4221af 100644 --- a/resources/services/backup/plans.go +++ b/resources/services/backup/plans.go @@ -3,14 +3,15 @@ package backup import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/backup" "github.com/aws/aws-sdk-go-v2/service/backup/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Plans() *schema.Table { diff --git a/resources/services/backup/plans_mock_test.go b/resources/services/backup/plans_mock_test.go index 9059d534..dbb76171 100644 --- a/resources/services/backup/plans_mock_test.go +++ b/resources/services/backup/plans_mock_test.go @@ -3,14 +3,15 @@ package backup import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/backup" "github.com/aws/aws-sdk-go-v2/service/backup/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildBackupPlansMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/backup/protected_resources.go b/resources/services/backup/protected_resources.go index 09ec7670..9ac77777 100644 --- a/resources/services/backup/protected_resources.go +++ b/resources/services/backup/protected_resources.go @@ -3,13 +3,14 @@ package backup import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/backup" "github.com/aws/aws-sdk-go-v2/service/backup/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ProtectedResources() *schema.Table { diff --git a/resources/services/backup/protected_resources_mock_test.go b/resources/services/backup/protected_resources_mock_test.go index 3563f80a..89afd6a1 100644 --- a/resources/services/backup/protected_resources_mock_test.go +++ b/resources/services/backup/protected_resources_mock_test.go @@ -3,13 +3,14 @@ package backup import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/backup" "github.com/aws/aws-sdk-go-v2/service/backup/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildProtectedResourcesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/backup/report_plans.go b/resources/services/backup/report_plans.go index e02adfa8..edeca26f 100644 --- a/resources/services/backup/report_plans.go +++ b/resources/services/backup/report_plans.go @@ -3,13 +3,14 @@ package backup import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/backup" "github.com/aws/aws-sdk-go-v2/service/backup/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func ReportPlans() *schema.Table { diff --git a/resources/services/backup/report_plans_mock_test.go b/resources/services/backup/report_plans_mock_test.go index f31a2bb4..2c40b166 100644 --- a/resources/services/backup/report_plans_mock_test.go +++ b/resources/services/backup/report_plans_mock_test.go @@ -3,13 +3,14 @@ package backup import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/backup" "github.com/aws/aws-sdk-go-v2/service/backup/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildReportPlansMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/backup/vault_recovery_points.go b/resources/services/backup/vault_recovery_points.go index 6f043f8b..70d75b72 100644 --- a/resources/services/backup/vault_recovery_points.go +++ b/resources/services/backup/vault_recovery_points.go @@ -4,15 +4,16 @@ import ( "context" "strings" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/backup" "github.com/aws/aws-sdk-go-v2/service/backup/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func vaultRecoveryPoints() *schema.Table { diff --git a/resources/services/backup/vaults.go b/resources/services/backup/vaults.go index 4281c50d..78c40b47 100644 --- a/resources/services/backup/vaults.go +++ b/resources/services/backup/vaults.go @@ -5,14 +5,15 @@ import ( "encoding/json" "strings" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/backup" "github.com/aws/aws-sdk-go-v2/service/backup/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Vaults() *schema.Table { diff --git a/resources/services/backup/vaults_mock_test.go b/resources/services/backup/vaults_mock_test.go index 2262eb86..8ac6458a 100644 --- a/resources/services/backup/vaults_mock_test.go +++ b/resources/services/backup/vaults_mock_test.go @@ -3,14 +3,15 @@ package backup import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/backup" "github.com/aws/aws-sdk-go-v2/service/backup/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildBackupVaultsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/batch/compute_environments.go b/resources/services/batch/compute_environments.go index a9970de2..f2d18c2c 100644 --- a/resources/services/batch/compute_environments.go +++ b/resources/services/batch/compute_environments.go @@ -3,14 +3,15 @@ package batch import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/batch" "github.com/aws/aws-sdk-go-v2/service/batch/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func ComputeEnvironments() *schema.Table { diff --git a/resources/services/batch/compute_environments_mock_test.go b/resources/services/batch/compute_environments_mock_test.go index e7bf42ab..fe4b2beb 100644 --- a/resources/services/batch/compute_environments_mock_test.go +++ b/resources/services/batch/compute_environments_mock_test.go @@ -3,13 +3,14 @@ package batch import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/batch" "github.com/aws/aws-sdk-go-v2/service/batch/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildBatchComputeEnvironmentsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/batch/job_definitions.go b/resources/services/batch/job_definitions.go index ac15dedf..dec7853b 100644 --- a/resources/services/batch/job_definitions.go +++ b/resources/services/batch/job_definitions.go @@ -3,14 +3,15 @@ package batch import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/batch" "github.com/aws/aws-sdk-go-v2/service/batch/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func JobDefinitions() *schema.Table { diff --git a/resources/services/batch/job_definitions_mock_test.go b/resources/services/batch/job_definitions_mock_test.go index 7297d6cc..45a7b3ce 100644 --- a/resources/services/batch/job_definitions_mock_test.go +++ b/resources/services/batch/job_definitions_mock_test.go @@ -3,13 +3,14 @@ package batch import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/batch" "github.com/aws/aws-sdk-go-v2/service/batch/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildBatchJobDefinitionsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/batch/job_queues.go b/resources/services/batch/job_queues.go index f8024eb4..b3d5583f 100644 --- a/resources/services/batch/job_queues.go +++ b/resources/services/batch/job_queues.go @@ -3,14 +3,15 @@ package batch import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/batch" "github.com/aws/aws-sdk-go-v2/service/batch/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func JobQueues() *schema.Table { diff --git a/resources/services/batch/job_queues_mock_test.go b/resources/services/batch/job_queues_mock_test.go index c0c9971b..a541efc0 100644 --- a/resources/services/batch/job_queues_mock_test.go +++ b/resources/services/batch/job_queues_mock_test.go @@ -3,13 +3,14 @@ package batch import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/batch" "github.com/aws/aws-sdk-go-v2/service/batch/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildBatchJobQueuesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/batch/jobs.go b/resources/services/batch/jobs.go index 3f80f0ff..f9fa65be 100644 --- a/resources/services/batch/jobs.go +++ b/resources/services/batch/jobs.go @@ -3,14 +3,15 @@ package batch import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/batch" "github.com/aws/aws-sdk-go-v2/service/batch/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) var allJobStatuses = []types.JobStatus{ diff --git a/resources/services/batch/jobs_mock_test.go b/resources/services/batch/jobs_mock_test.go index f21a102b..78f14e99 100644 --- a/resources/services/batch/jobs_mock_test.go +++ b/resources/services/batch/jobs_mock_test.go @@ -3,13 +3,14 @@ package batch import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/batch" "github.com/aws/aws-sdk-go-v2/service/batch/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildBatchJobsMock(t *testing.T, m *mocks.MockBatchClient) client.Services { diff --git a/resources/services/cloudformation/models/stacksets.go b/resources/services/cloudformation/models/stacksets.go index 0a03245d..52947b21 100644 --- a/resources/services/cloudformation/models/stacksets.go +++ b/resources/services/cloudformation/models/stacksets.go @@ -11,14 +11,17 @@ type ExpandedStackSet struct { types.StackSet types.CallAs } + type ExpandedStackSetOperationSummary struct { types.StackSetOperationSummary types.CallAs } + type ExpandedStackSetOperation struct { types.StackSetOperation types.CallAs } + type ExpandedStackInstanceSummary struct { types.StackInstanceSummary types.CallAs diff --git a/resources/services/cloudformation/stack_instance_resource_drifts.go b/resources/services/cloudformation/stack_instance_resource_drifts.go index 3b857716..4bc328bc 100644 --- a/resources/services/cloudformation/stack_instance_resource_drifts.go +++ b/resources/services/cloudformation/stack_instance_resource_drifts.go @@ -3,14 +3,15 @@ package cloudformation import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/cloudformation/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudformation" "github.com/aws/aws-sdk-go-v2/service/cloudformation/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/cloudformation/models" ) func stackInstanceResourceDrifts() *schema.Table { @@ -39,6 +40,7 @@ The 'request_account_id' and 'request_region' columns are added to show the acco }, } } + func fetchStackInstanceResourceDrifts(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { instance := parent.Item.(models.ExpandedStackInstanceSummary) if instance.DriftStatus == types.StackDriftStatusNotChecked { diff --git a/resources/services/cloudformation/stack_instances.go b/resources/services/cloudformation/stack_instances.go index fe3e0c4d..d924ac2b 100644 --- a/resources/services/cloudformation/stack_instances.go +++ b/resources/services/cloudformation/stack_instances.go @@ -3,13 +3,14 @@ package cloudformation import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/cloudformation/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudformation" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/cloudformation/models" ) func stackInstanceSummaries() *schema.Table { @@ -46,6 +47,7 @@ func stackInstanceSummaries() *schema.Table { }, } } + func fetchStackInstanceSummary(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { stack := parent.Item.(models.ExpandedStackSet) diff --git a/resources/services/cloudformation/stack_resources.go b/resources/services/cloudformation/stack_resources.go index a19569ea..ee46d25d 100644 --- a/resources/services/cloudformation/stack_resources.go +++ b/resources/services/cloudformation/stack_resources.go @@ -3,12 +3,13 @@ package cloudformation import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/cloudformation" "github.com/aws/aws-sdk-go-v2/service/cloudformation/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func stackResources() *schema.Table { diff --git a/resources/services/cloudformation/stack_templates.go b/resources/services/cloudformation/stack_templates.go index 93e83936..376870a2 100644 --- a/resources/services/cloudformation/stack_templates.go +++ b/resources/services/cloudformation/stack_templates.go @@ -5,15 +5,15 @@ import ( "encoding/json" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/cloudformation" "github.com/aws/aws-sdk-go-v2/service/cloudformation/types" - - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - cqtypes "github.com/cloudquery/plugin-sdk/v4/types" "github.com/ghodss/yaml" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + cqtypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func stackTemplates() *schema.Table { diff --git a/resources/services/cloudformation/stacks.go b/resources/services/cloudformation/stacks.go index 43427566..e3a4f13a 100644 --- a/resources/services/cloudformation/stacks.go +++ b/resources/services/cloudformation/stacks.go @@ -3,13 +3,14 @@ package cloudformation import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/cloudformation" "github.com/aws/aws-sdk-go-v2/service/cloudformation/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Stacks() *schema.Table { diff --git a/resources/services/cloudformation/stacks_mock_test.go b/resources/services/cloudformation/stacks_mock_test.go index 80a166ca..4580d7b4 100644 --- a/resources/services/cloudformation/stacks_mock_test.go +++ b/resources/services/cloudformation/stacks_mock_test.go @@ -3,14 +3,15 @@ package cloudformation import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudformation" "github.com/aws/aws-sdk-go-v2/service/cloudformation/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildStacksWithTemplate(tmpl string) func(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/cloudformation/stackset_mock_test.go b/resources/services/cloudformation/stackset_mock_test.go index cff891ba..936970fc 100644 --- a/resources/services/cloudformation/stackset_mock_test.go +++ b/resources/services/cloudformation/stackset_mock_test.go @@ -3,13 +3,14 @@ package cloudformation import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/cloudformation" "github.com/aws/aws-sdk-go-v2/service/cloudformation/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildStackSet(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/cloudformation/stackset_operation_results.go b/resources/services/cloudformation/stackset_operation_results.go index b2830171..99779f8d 100644 --- a/resources/services/cloudformation/stackset_operation_results.go +++ b/resources/services/cloudformation/stackset_operation_results.go @@ -3,13 +3,14 @@ package cloudformation import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/cloudformation/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/cloudformation" "github.com/aws/aws-sdk-go-v2/service/cloudformation/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/cloudformation/models" ) func stackSetOperationResults() *schema.Table { @@ -39,6 +40,7 @@ The 'request_account_id' and 'request_region' columns are added to show the acco }, } } + func fetchCloudformationStackSetOperationResults(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { stackSet := parent.Parent.Item.(models.ExpandedStackSet) operation := parent.Item.(models.ExpandedStackSetOperation) diff --git a/resources/services/cloudformation/stackset_operations.go b/resources/services/cloudformation/stackset_operations.go index a5981b74..243f2ccb 100644 --- a/resources/services/cloudformation/stackset_operations.go +++ b/resources/services/cloudformation/stackset_operations.go @@ -3,12 +3,13 @@ package cloudformation import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/cloudformation/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/cloudformation" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/cloudformation/models" ) func stackSetOperations() *schema.Table { @@ -43,6 +44,7 @@ func stackSetOperations() *schema.Table { }, } } + func fetchCloudformationStackSetOperations(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { stack := parent.Item.(models.ExpandedStackSet) diff --git a/resources/services/cloudformation/stacksets.go b/resources/services/cloudformation/stacksets.go index bf146a53..96ba80c8 100644 --- a/resources/services/cloudformation/stacksets.go +++ b/resources/services/cloudformation/stacksets.go @@ -3,14 +3,15 @@ package cloudformation import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/cloudformation/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/cloudformation" "github.com/aws/aws-sdk-go-v2/service/cloudformation/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/cloudformation/models" ) func StackSets() *schema.Table { diff --git a/resources/services/cloudformation/template_summaries.go b/resources/services/cloudformation/template_summaries.go index 97dc0a2b..cd342980 100644 --- a/resources/services/cloudformation/template_summaries.go +++ b/resources/services/cloudformation/template_summaries.go @@ -3,13 +3,14 @@ package cloudformation import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/cloudformation" "github.com/aws/aws-sdk-go-v2/service/cloudformation/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - cqtypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + cqtypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func templateSummaries() *schema.Table { diff --git a/resources/services/cloudfront/cache_policies.go b/resources/services/cloudfront/cache_policies.go index 059dbacc..64bbb0dd 100644 --- a/resources/services/cloudfront/cache_policies.go +++ b/resources/services/cloudfront/cache_policies.go @@ -3,13 +3,14 @@ package cloudfront import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudfront" "github.com/aws/aws-sdk-go-v2/service/cloudfront/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func CachePolicies() *schema.Table { diff --git a/resources/services/cloudfront/cache_policies_mock_test.go b/resources/services/cloudfront/cache_policies_mock_test.go index 6b38b7cd..70b0bc08 100644 --- a/resources/services/cloudfront/cache_policies_mock_test.go +++ b/resources/services/cloudfront/cache_policies_mock_test.go @@ -3,13 +3,14 @@ package cloudfront import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/cloudfront" cloudfrontTypes "github.com/aws/aws-sdk-go-v2/service/cloudfront/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildCloudfrontCachePoliciesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/cloudfront/distributions.go b/resources/services/cloudfront/distributions.go index 983a3bda..dc397438 100644 --- a/resources/services/cloudfront/distributions.go +++ b/resources/services/cloudfront/distributions.go @@ -3,13 +3,14 @@ package cloudfront import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/cloudfront" "github.com/aws/aws-sdk-go-v2/service/cloudfront/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Distributions() *schema.Table { diff --git a/resources/services/cloudfront/distributions_mock_test.go b/resources/services/cloudfront/distributions_mock_test.go index 54d3a42a..8d9ce9d3 100644 --- a/resources/services/cloudfront/distributions_mock_test.go +++ b/resources/services/cloudfront/distributions_mock_test.go @@ -3,13 +3,14 @@ package cloudfront import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/cloudfront" cloudfrontTypes "github.com/aws/aws-sdk-go-v2/service/cloudfront/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildCloudfrontDistributionsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/cloudfront/functions.go b/resources/services/cloudfront/functions.go index 6108f4f3..a9dab4e6 100644 --- a/resources/services/cloudfront/functions.go +++ b/resources/services/cloudfront/functions.go @@ -3,13 +3,14 @@ package cloudfront import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudfront" "github.com/aws/aws-sdk-go-v2/service/cloudfront/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Functions() *schema.Table { @@ -63,6 +64,7 @@ func fetchFunctions(ctx context.Context, meta schema.ClientMeta, parent *schema. } return nil } + func getFunction(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource) error { cl := meta.(*client.Client) svc := cl.Services(client.AWSServiceCloudfront).Cloudfront diff --git a/resources/services/cloudfront/functions_mock_test.go b/resources/services/cloudfront/functions_mock_test.go index 3abdc002..b0e77553 100644 --- a/resources/services/cloudfront/functions_mock_test.go +++ b/resources/services/cloudfront/functions_mock_test.go @@ -3,13 +3,14 @@ package cloudfront import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/cloudfront" cloudfrontTypes "github.com/aws/aws-sdk-go-v2/service/cloudfront/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildCloudfronFunctionsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/cloudhsmv2/backups.go b/resources/services/cloudhsmv2/backups.go index f7d83b89..ebcc1485 100644 --- a/resources/services/cloudhsmv2/backups.go +++ b/resources/services/cloudhsmv2/backups.go @@ -3,15 +3,16 @@ package cloudhsmv2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/cloudhsmv2" "github.com/aws/aws-sdk-go-v2/service/cloudhsmv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Backups() *schema.Table { diff --git a/resources/services/cloudhsmv2/backups_mock_test.go b/resources/services/cloudhsmv2/backups_mock_test.go index e20f2651..4cfb4df4 100644 --- a/resources/services/cloudhsmv2/backups_mock_test.go +++ b/resources/services/cloudhsmv2/backups_mock_test.go @@ -3,13 +3,14 @@ package cloudhsmv2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/cloudhsmv2" "github.com/aws/aws-sdk-go-v2/service/cloudhsmv2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildHSMBackups(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/cloudhsmv2/clusters.go b/resources/services/cloudhsmv2/clusters.go index e0fafc21..d63e511e 100644 --- a/resources/services/cloudhsmv2/clusters.go +++ b/resources/services/cloudhsmv2/clusters.go @@ -3,15 +3,16 @@ package cloudhsmv2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/cloudhsmv2" "github.com/aws/aws-sdk-go-v2/service/cloudhsmv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Clusters() *schema.Table { diff --git a/resources/services/cloudhsmv2/clusters_mock_test.go b/resources/services/cloudhsmv2/clusters_mock_test.go index 3c3f942d..763f5154 100644 --- a/resources/services/cloudhsmv2/clusters_mock_test.go +++ b/resources/services/cloudhsmv2/clusters_mock_test.go @@ -3,13 +3,14 @@ package cloudhsmv2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/cloudhsmv2" "github.com/aws/aws-sdk-go-v2/service/cloudhsmv2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildHSMClusters(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/cloudtrail/channels.go b/resources/services/cloudtrail/channels.go index 0d2f8fc1..3a18deff 100644 --- a/resources/services/cloudtrail/channels.go +++ b/resources/services/cloudtrail/channels.go @@ -4,12 +4,12 @@ import ( "context" "github.com/apache/arrow/go/v16/arrow" - - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/service/cloudtrail" "github.com/aws/aws-sdk-go-v2/service/cloudtrail/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Channels() *schema.Table { diff --git a/resources/services/cloudtrail/channels_mock_test.go b/resources/services/cloudtrail/channels_mock_test.go index 2bb8a6c7..ff535e3f 100644 --- a/resources/services/cloudtrail/channels_mock_test.go +++ b/resources/services/cloudtrail/channels_mock_test.go @@ -3,13 +3,14 @@ package cloudtrail import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/cloudtrail" "github.com/aws/aws-sdk-go-v2/service/cloudtrail/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildCloudtrailChannelsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/cloudtrail/trail_event_selectors.go b/resources/services/cloudtrail/trail_event_selectors.go index 470a23c1..6d1f539d 100644 --- a/resources/services/cloudtrail/trail_event_selectors.go +++ b/resources/services/cloudtrail/trail_event_selectors.go @@ -3,11 +3,12 @@ package cloudtrail import ( "context" + "github.com/aws/aws-sdk-go-v2/service/cloudtrail" + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" "github.com/OpsHelmInc/cloudquery/v2/resources/services/cloudtrail/models" - "github.com/aws/aws-sdk-go-v2/service/cloudtrail" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" ) func trailEventSelectors() *schema.Table { diff --git a/resources/services/cloudtrail/trails.go b/resources/services/cloudtrail/trails.go index d96d13f0..2d497a5f 100644 --- a/resources/services/cloudtrail/trails.go +++ b/resources/services/cloudtrail/trails.go @@ -5,16 +5,17 @@ import ( "fmt" "regexp" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/cloudtrail/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/cloudtrail" "github.com/aws/aws-sdk-go-v2/service/cloudtrail/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/cloudtrail/models" ) func Trails() *schema.Table { diff --git a/resources/services/cloudtrail/trails_mock_test.go b/resources/services/cloudtrail/trails_mock_test.go index 4b2d89c2..951b2041 100644 --- a/resources/services/cloudtrail/trails_mock_test.go +++ b/resources/services/cloudtrail/trails_mock_test.go @@ -3,14 +3,15 @@ package cloudtrail import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudtrail" "github.com/aws/aws-sdk-go-v2/service/cloudtrail/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildCloudtrailTrailsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/cloudwatch/alarms.go b/resources/services/cloudwatch/alarms.go index c0c1b44d..f2519dad 100644 --- a/resources/services/cloudwatch/alarms.go +++ b/resources/services/cloudwatch/alarms.go @@ -3,13 +3,14 @@ package cloudwatch import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Alarms() *schema.Table { @@ -60,6 +61,7 @@ func fetchCloudwatchAlarms(ctx context.Context, meta schema.ClientMeta, parent * } return nil } + func resolveCloudwatchAlarmDimensions(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { alarm := resource.Item.(types.MetricAlarm) dimensions := make(map[string]*string) diff --git a/resources/services/cloudwatch/alarms_mock_test.go b/resources/services/cloudwatch/alarms_mock_test.go index bf63d5f6..12bd3ad6 100644 --- a/resources/services/cloudwatch/alarms_mock_test.go +++ b/resources/services/cloudwatch/alarms_mock_test.go @@ -3,13 +3,14 @@ package cloudwatch import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildCloudWatchAlarmsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/cloudwatchlogs/data_protection_policies.go b/resources/services/cloudwatchlogs/data_protection_policies.go index 76268ea4..4736c4a5 100644 --- a/resources/services/cloudwatchlogs/data_protection_policies.go +++ b/resources/services/cloudwatchlogs/data_protection_policies.go @@ -3,12 +3,13 @@ package cloudwatchlogs import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func dataProtectionPolicy() *schema.Table { @@ -31,6 +32,7 @@ func dataProtectionPolicy() *schema.Table { }, } } + func fetchDataProtectionPolicy(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { lg := parent.Item.(types.LogGroup) if lg.DataProtectionStatus == "" { // Inactive Data Protection policy, don't attempt to fetch diff --git a/resources/services/cloudwatchlogs/log_groups.go b/resources/services/cloudwatchlogs/log_groups.go index f36cc943..3488a46e 100644 --- a/resources/services/cloudwatchlogs/log_groups.go +++ b/resources/services/cloudwatchlogs/log_groups.go @@ -3,13 +3,14 @@ package cloudwatchlogs import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func LogGroups() *schema.Table { diff --git a/resources/services/cloudwatchlogs/log_groups_mock_test.go b/resources/services/cloudwatchlogs/log_groups_mock_test.go index e330a1d6..c2c763b1 100644 --- a/resources/services/cloudwatchlogs/log_groups_mock_test.go +++ b/resources/services/cloudwatchlogs/log_groups_mock_test.go @@ -3,13 +3,14 @@ package cloudwatchlogs import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildCloudwatchLogsLogGroupsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/cloudwatchlogs/subscription_filters.go b/resources/services/cloudwatchlogs/subscription_filters.go index 73f13261..f800be48 100644 --- a/resources/services/cloudwatchlogs/subscription_filters.go +++ b/resources/services/cloudwatchlogs/subscription_filters.go @@ -3,12 +3,13 @@ package cloudwatchlogs import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func subscriptionFilters() *schema.Table { @@ -31,6 +32,7 @@ func subscriptionFilters() *schema.Table { }, } } + func fetchCloudwatchlogsSubscriptionFilters(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { config := cloudwatchlogs.DescribeSubscriptionFiltersInput{ LogGroupName: parent.Item.(types.LogGroup).LogGroupName, diff --git a/resources/services/codeartifact/domains.go b/resources/services/codeartifact/domains.go index 5f228f28..51d1aacc 100644 --- a/resources/services/codeartifact/domains.go +++ b/resources/services/codeartifact/domains.go @@ -3,13 +3,14 @@ package codeartifact import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/codeartifact" "github.com/aws/aws-sdk-go-v2/service/codeartifact/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Domains() *schema.Table { diff --git a/resources/services/codeartifact/domains_mock_test.go b/resources/services/codeartifact/domains_mock_test.go index 295acbce..33feb975 100644 --- a/resources/services/codeartifact/domains_mock_test.go +++ b/resources/services/codeartifact/domains_mock_test.go @@ -3,13 +3,14 @@ package codeartifact import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/codeartifact" "github.com/aws/aws-sdk-go-v2/service/codeartifact/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDomains(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/codeartifact/helpers.go b/resources/services/codeartifact/helpers.go index 463c722e..53b805a6 100644 --- a/resources/services/codeartifact/helpers.go +++ b/resources/services/codeartifact/helpers.go @@ -3,11 +3,11 @@ package codeartifact import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/service/codeartifact" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/thoas/go-funk" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" ) func resolveCodeartifactTags(path string) schema.ColumnResolver { diff --git a/resources/services/codeartifact/repositories.go b/resources/services/codeartifact/repositories.go index 03581864..54c00ca2 100644 --- a/resources/services/codeartifact/repositories.go +++ b/resources/services/codeartifact/repositories.go @@ -3,13 +3,14 @@ package codeartifact import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/codeartifact" "github.com/aws/aws-sdk-go-v2/service/codeartifact/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Repositories() *schema.Table { diff --git a/resources/services/codeartifact/repositories_mock_test.go b/resources/services/codeartifact/repositories_mock_test.go index 321c331d..2f4a2d54 100644 --- a/resources/services/codeartifact/repositories_mock_test.go +++ b/resources/services/codeartifact/repositories_mock_test.go @@ -3,13 +3,14 @@ package codeartifact import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/codeartifact" "github.com/aws/aws-sdk-go-v2/service/codeartifact/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRepositories(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/codebuild/builds.go b/resources/services/codebuild/builds.go index 2209b429..378ea23d 100644 --- a/resources/services/codebuild/builds.go +++ b/resources/services/codebuild/builds.go @@ -3,11 +3,12 @@ package codebuild import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/service/codebuild" "github.com/aws/aws-sdk-go-v2/service/codebuild/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func builds() *schema.Table { diff --git a/resources/services/codebuild/projects.go b/resources/services/codebuild/projects.go index 026e9d66..8c1eae20 100644 --- a/resources/services/codebuild/projects.go +++ b/resources/services/codebuild/projects.go @@ -3,13 +3,14 @@ package codebuild import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/codebuild" "github.com/aws/aws-sdk-go-v2/service/codebuild/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Projects() *schema.Table { diff --git a/resources/services/codebuild/projects_mock_test.go b/resources/services/codebuild/projects_mock_test.go index 62c59214..e85ad021 100644 --- a/resources/services/codebuild/projects_mock_test.go +++ b/resources/services/codebuild/projects_mock_test.go @@ -3,12 +3,13 @@ package codebuild import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/codebuild" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildCodebuildProjects(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/codebuild/source_credentials.go b/resources/services/codebuild/source_credentials.go index f334f00b..3d2c8861 100644 --- a/resources/services/codebuild/source_credentials.go +++ b/resources/services/codebuild/source_credentials.go @@ -3,12 +3,13 @@ package codebuild import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/codebuild" "github.com/aws/aws-sdk-go-v2/service/codebuild/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func SourceCredentials() *schema.Table { diff --git a/resources/services/codebuild/source_credentials_mock_test.go b/resources/services/codebuild/source_credentials_mock_test.go index d97196af..e84da619 100644 --- a/resources/services/codebuild/source_credentials_mock_test.go +++ b/resources/services/codebuild/source_credentials_mock_test.go @@ -3,12 +3,13 @@ package codebuild import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/codebuild" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildSourceCredentials(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/codecommit/repositories.go b/resources/services/codecommit/repositories.go index 5083f1a7..9df754b5 100644 --- a/resources/services/codecommit/repositories.go +++ b/resources/services/codecommit/repositories.go @@ -3,14 +3,15 @@ package codecommit import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/codecommit" "github.com/aws/aws-sdk-go-v2/service/codecommit/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Repositories() *schema.Table { diff --git a/resources/services/codecommit/repositories_mock_test.go b/resources/services/codecommit/repositories_mock_test.go index 3ef7a6a1..a2222168 100644 --- a/resources/services/codecommit/repositories_mock_test.go +++ b/resources/services/codecommit/repositories_mock_test.go @@ -3,13 +3,14 @@ package codecommit import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/codecommit" "github.com/aws/aws-sdk-go-v2/service/codecommit/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRepositories(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/codepipeline/pipelines.go b/resources/services/codepipeline/pipelines.go index 2f5d25b4..80d9b67d 100644 --- a/resources/services/codepipeline/pipelines.go +++ b/resources/services/codepipeline/pipelines.go @@ -3,14 +3,15 @@ package codepipeline import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/codepipeline" "github.com/aws/aws-sdk-go-v2/service/codepipeline/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Pipelines() *schema.Table { diff --git a/resources/services/codepipeline/pipelines_mock_test.go b/resources/services/codepipeline/pipelines_mock_test.go index e5853415..045e890b 100644 --- a/resources/services/codepipeline/pipelines_mock_test.go +++ b/resources/services/codepipeline/pipelines_mock_test.go @@ -3,13 +3,14 @@ package codepipeline import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/codepipeline" "github.com/aws/aws-sdk-go-v2/service/codepipeline/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildPipelines(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/codepipeline/webhooks.go b/resources/services/codepipeline/webhooks.go index 919bd146..3f620a48 100644 --- a/resources/services/codepipeline/webhooks.go +++ b/resources/services/codepipeline/webhooks.go @@ -3,13 +3,14 @@ package codepipeline import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/codepipeline" "github.com/aws/aws-sdk-go-v2/service/codepipeline/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Webhooks() *schema.Table { diff --git a/resources/services/codepipeline/webhooks_mock_test.go b/resources/services/codepipeline/webhooks_mock_test.go index 3749c684..a6f534b8 100644 --- a/resources/services/codepipeline/webhooks_mock_test.go +++ b/resources/services/codepipeline/webhooks_mock_test.go @@ -3,13 +3,14 @@ package codepipeline import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/codepipeline" "github.com/aws/aws-sdk-go-v2/service/codepipeline/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildWebhooks(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/cognito/identity_pools.go b/resources/services/cognito/identity_pools.go index bba4140b..90d4e3e1 100644 --- a/resources/services/cognito/identity_pools.go +++ b/resources/services/cognito/identity_pools.go @@ -3,14 +3,15 @@ package cognito import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cognitoidentity" "github.com/aws/aws-sdk-go-v2/service/cognitoidentity/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func IdentityPools() *schema.Table { diff --git a/resources/services/cognito/identity_pools_mock_test.go b/resources/services/cognito/identity_pools_mock_test.go index a39757e2..6905091a 100644 --- a/resources/services/cognito/identity_pools_mock_test.go +++ b/resources/services/cognito/identity_pools_mock_test.go @@ -3,14 +3,15 @@ package cognito import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cognitoidentity" "github.com/aws/aws-sdk-go-v2/service/cognitoidentity/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildCognitoIdentityPools(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/cognito/user_pool_identity_providers.go b/resources/services/cognito/user_pool_identity_providers.go index db1fa733..48b5ea83 100644 --- a/resources/services/cognito/user_pool_identity_providers.go +++ b/resources/services/cognito/user_pool_identity_providers.go @@ -3,12 +3,13 @@ package cognito import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/cognitoidentityprovider" "github.com/aws/aws-sdk-go-v2/service/cognitoidentityprovider/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func userPoolIdentityProviders() *schema.Table { diff --git a/resources/services/cognito/user_pools.go b/resources/services/cognito/user_pools.go index aaed4a7b..bca77a2b 100644 --- a/resources/services/cognito/user_pools.go +++ b/resources/services/cognito/user_pools.go @@ -3,14 +3,15 @@ package cognito import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cognitoidentityprovider" "github.com/aws/aws-sdk-go-v2/service/cognitoidentityprovider/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func UserPools() *schema.Table { diff --git a/resources/services/cognito/user_pools_mock_test.go b/resources/services/cognito/user_pools_mock_test.go index c22e37f7..b4d3c578 100644 --- a/resources/services/cognito/user_pools_mock_test.go +++ b/resources/services/cognito/user_pools_mock_test.go @@ -3,14 +3,15 @@ package cognito import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cognitoidentityprovider" "github.com/aws/aws-sdk-go-v2/service/cognitoidentityprovider/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildCognitoUserPools(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/config/config_rule_compliance_details.go b/resources/services/config/config_rule_compliance_details.go index 74df764e..df78554d 100644 --- a/resources/services/config/config_rule_compliance_details.go +++ b/resources/services/config/config_rule_compliance_details.go @@ -3,12 +3,13 @@ package config import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/configservice" "github.com/aws/aws-sdk-go-v2/service/configservice/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func configRuleComplianceDetails() *schema.Table { diff --git a/resources/services/config/config_rule_compliance_details_mock_test.go b/resources/services/config/config_rule_compliance_details_mock_test.go index 1132492a..f8e493af 100644 --- a/resources/services/config/config_rule_compliance_details_mock_test.go +++ b/resources/services/config/config_rule_compliance_details_mock_test.go @@ -3,13 +3,14 @@ package config import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/configservice" "github.com/aws/aws-sdk-go-v2/service/configservice/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildComplianceDetails(t *testing.T, m *mocks.MockConfigserviceClient) client.Services { diff --git a/resources/services/config/config_rule_compliances.go b/resources/services/config/config_rule_compliances.go index e9440481..144224d8 100644 --- a/resources/services/config/config_rule_compliances.go +++ b/resources/services/config/config_rule_compliances.go @@ -3,13 +3,14 @@ package config import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/configservice" "github.com/aws/aws-sdk-go-v2/service/configservice/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func configRuleCompliances() *schema.Table { diff --git a/resources/services/config/config_rules.go b/resources/services/config/config_rules.go index d8a4b1b0..3d571848 100644 --- a/resources/services/config/config_rules.go +++ b/resources/services/config/config_rules.go @@ -3,12 +3,13 @@ package config import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/configservice" "github.com/aws/aws-sdk-go-v2/service/configservice/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ConfigRules() *schema.Table { diff --git a/resources/services/config/config_rules_mock_test.go b/resources/services/config/config_rules_mock_test.go index af1c0324..d39cc1bf 100644 --- a/resources/services/config/config_rules_mock_test.go +++ b/resources/services/config/config_rules_mock_test.go @@ -3,13 +3,14 @@ package config import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/configservice" "github.com/aws/aws-sdk-go-v2/service/configservice/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildConfigRules(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/config/configuration_aggregators.go b/resources/services/config/configuration_aggregators.go index 5bc0eb56..96ae71ad 100644 --- a/resources/services/config/configuration_aggregators.go +++ b/resources/services/config/configuration_aggregators.go @@ -3,12 +3,13 @@ package config import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/configservice" "github.com/aws/aws-sdk-go-v2/service/configservice/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ConfigurationAggregators() *schema.Table { diff --git a/resources/services/config/configuration_aggregators_mock_test.go b/resources/services/config/configuration_aggregators_mock_test.go index f9d34bf2..b337208e 100644 --- a/resources/services/config/configuration_aggregators_mock_test.go +++ b/resources/services/config/configuration_aggregators_mock_test.go @@ -3,13 +3,14 @@ package config import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/configservice" "github.com/aws/aws-sdk-go-v2/service/configservice/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildConfigurationAggregators(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/config/configuration_recorders.go b/resources/services/config/configuration_recorders.go index fa6bd8f1..d44fc314 100644 --- a/resources/services/config/configuration_recorders.go +++ b/resources/services/config/configuration_recorders.go @@ -4,15 +4,16 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/config/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/configservice" "github.com/aws/aws-sdk-go-v2/service/configservice/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/config/models" ) func ConfigurationRecorders() *schema.Table { diff --git a/resources/services/config/configuration_recorders_mock_test.go b/resources/services/config/configuration_recorders_mock_test.go index 1e9df886..8693c1ad 100644 --- a/resources/services/config/configuration_recorders_mock_test.go +++ b/resources/services/config/configuration_recorders_mock_test.go @@ -3,13 +3,14 @@ package config import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/configservice" "github.com/aws/aws-sdk-go-v2/service/configservice/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildConfigConfigurationRecorders(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/config/conformance_pack_mock_test.go b/resources/services/config/conformance_pack_mock_test.go index c237a4f3..fc515a30 100644 --- a/resources/services/config/conformance_pack_mock_test.go +++ b/resources/services/config/conformance_pack_mock_test.go @@ -3,13 +3,14 @@ package config import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/configservice" "github.com/aws/aws-sdk-go-v2/service/configservice/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildConfigConformancePack(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/config/conformance_pack_rule_compliances.go b/resources/services/config/conformance_pack_rule_compliances.go index ec8da56f..e9cc2cad 100644 --- a/resources/services/config/conformance_pack_rule_compliances.go +++ b/resources/services/config/conformance_pack_rule_compliances.go @@ -3,13 +3,14 @@ package config import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/config/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/configservice" "github.com/aws/aws-sdk-go-v2/service/configservice/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/config/models" ) func conformancePackRuleCompliances() *schema.Table { diff --git a/resources/services/config/conformance_packs.go b/resources/services/config/conformance_packs.go index 51dd9c89..1a4a1b81 100644 --- a/resources/services/config/conformance_packs.go +++ b/resources/services/config/conformance_packs.go @@ -4,13 +4,14 @@ import ( "context" "errors" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/configservice" "github.com/aws/aws-sdk-go-v2/service/configservice/types" "github.com/aws/smithy-go" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ConformancePacks() *schema.Table { diff --git a/resources/services/config/remediation_configurations.go b/resources/services/config/remediation_configurations.go index 12ff7603..f259972b 100644 --- a/resources/services/config/remediation_configurations.go +++ b/resources/services/config/remediation_configurations.go @@ -3,12 +3,13 @@ package config import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/configservice" "github.com/aws/aws-sdk-go-v2/service/configservice/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func remediationConfigurations() *schema.Table { diff --git a/resources/services/config/remediation_configurations_mock_test.go b/resources/services/config/remediation_configurations_mock_test.go index 24d16aac..4533e9ef 100644 --- a/resources/services/config/remediation_configurations_mock_test.go +++ b/resources/services/config/remediation_configurations_mock_test.go @@ -3,13 +3,14 @@ package config import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/configservice" "github.com/aws/aws-sdk-go-v2/service/configservice/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRemediationConfigurations(t *testing.T, m *mocks.MockConfigserviceClient) client.Services { diff --git a/resources/services/dax/clusters.go b/resources/services/dax/clusters.go index 1d9b6d19..cb264303 100644 --- a/resources/services/dax/clusters.go +++ b/resources/services/dax/clusters.go @@ -3,14 +3,15 @@ package dax import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/dax" "github.com/aws/aws-sdk-go-v2/service/dax/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Clusters() *schema.Table { @@ -63,6 +64,7 @@ func fetchDaxClusters(ctx context.Context, meta schema.ClientMeta, parent *schem return nil } + func resolveClusterTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { cluster := resource.Item.(types.Cluster) diff --git a/resources/services/dax/clusters_mock_test.go b/resources/services/dax/clusters_mock_test.go index a7c1e79b..34e9332f 100644 --- a/resources/services/dax/clusters_mock_test.go +++ b/resources/services/dax/clusters_mock_test.go @@ -3,13 +3,14 @@ package dax import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/dax" "github.com/aws/aws-sdk-go-v2/service/dax/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDAXClustersMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/detective/graphs.go b/resources/services/detective/graphs.go index 333aa3bf..3b7bc653 100644 --- a/resources/services/detective/graphs.go +++ b/resources/services/detective/graphs.go @@ -3,13 +3,14 @@ package detective import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/detective" "github.com/aws/aws-sdk-go-v2/service/detective/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Graphs() *schema.Table { @@ -60,6 +61,7 @@ func fetchGraphs(ctx context.Context, meta schema.ClientMeta, parent *schema.Res return nil } + func resolveGraphTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { graph := resource.Item.(types.Graph) cl := meta.(*client.Client) diff --git a/resources/services/detective/graphs_mock_test.go b/resources/services/detective/graphs_mock_test.go index b2ead6a2..55860b65 100644 --- a/resources/services/detective/graphs_mock_test.go +++ b/resources/services/detective/graphs_mock_test.go @@ -3,13 +3,14 @@ package detective import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/detective" "github.com/aws/aws-sdk-go-v2/service/detective/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildGraphs(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/detective/members.go b/resources/services/detective/members.go index a9eda932..a824c0f8 100644 --- a/resources/services/detective/members.go +++ b/resources/services/detective/members.go @@ -3,12 +3,13 @@ package detective import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/detective" "github.com/aws/aws-sdk-go-v2/service/detective/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func members() *schema.Table { diff --git a/resources/services/directconnect/connections.go b/resources/services/directconnect/connections.go index 225cdd0e..b378961f 100644 --- a/resources/services/directconnect/connections.go +++ b/resources/services/directconnect/connections.go @@ -3,13 +3,14 @@ package directconnect import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/directconnect" "github.com/aws/aws-sdk-go-v2/service/directconnect/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Connections() *schema.Table { diff --git a/resources/services/directconnect/connections_mock_test.go b/resources/services/directconnect/connections_mock_test.go index a1d1db2c..fc484d9a 100644 --- a/resources/services/directconnect/connections_mock_test.go +++ b/resources/services/directconnect/connections_mock_test.go @@ -3,13 +3,14 @@ package directconnect import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/directconnect" "github.com/aws/aws-sdk-go-v2/service/directconnect/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDirectconnectConnection(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/directconnect/gateway_associations.go b/resources/services/directconnect/gateway_associations.go index ac0bd5f0..9ae0e680 100644 --- a/resources/services/directconnect/gateway_associations.go +++ b/resources/services/directconnect/gateway_associations.go @@ -3,13 +3,14 @@ package directconnect import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/directconnect" "github.com/aws/aws-sdk-go-v2/service/directconnect/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func gatewayAssociations() *schema.Table { diff --git a/resources/services/directconnect/gateway_attachments.go b/resources/services/directconnect/gateway_attachments.go index f792e4f8..930ed598 100644 --- a/resources/services/directconnect/gateway_attachments.go +++ b/resources/services/directconnect/gateway_attachments.go @@ -3,13 +3,14 @@ package directconnect import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/directconnect" "github.com/aws/aws-sdk-go-v2/service/directconnect/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func gatewayAttachments() *schema.Table { diff --git a/resources/services/directconnect/gateway_mock_test.go b/resources/services/directconnect/gateway_mock_test.go index a1d1cecb..0de55678 100644 --- a/resources/services/directconnect/gateway_mock_test.go +++ b/resources/services/directconnect/gateway_mock_test.go @@ -3,13 +3,14 @@ package directconnect import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/directconnect" "github.com/aws/aws-sdk-go-v2/service/directconnect/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDirectconnectGatewaysMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/directconnect/gateways.go b/resources/services/directconnect/gateways.go index 2ad0f4d9..c6ef4d19 100644 --- a/resources/services/directconnect/gateways.go +++ b/resources/services/directconnect/gateways.go @@ -4,14 +4,15 @@ import ( "context" "strings" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/directconnect" "github.com/aws/aws-sdk-go-v2/service/directconnect/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Gateways() *schema.Table { diff --git a/resources/services/directconnect/lags.go b/resources/services/directconnect/lags.go index 36c9e755..698b8ac7 100644 --- a/resources/services/directconnect/lags.go +++ b/resources/services/directconnect/lags.go @@ -3,13 +3,14 @@ package directconnect import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/directconnect" "github.com/aws/aws-sdk-go-v2/service/directconnect/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Lags() *schema.Table { diff --git a/resources/services/directconnect/lags_mock_test.go b/resources/services/directconnect/lags_mock_test.go index ece4c285..dedfddec 100644 --- a/resources/services/directconnect/lags_mock_test.go +++ b/resources/services/directconnect/lags_mock_test.go @@ -3,13 +3,14 @@ package directconnect import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/directconnect" "github.com/aws/aws-sdk-go-v2/service/directconnect/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDirectconnectLag(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/directconnect/virtual_interfaces.go b/resources/services/directconnect/virtual_interfaces.go index fc1a8772..025ae256 100644 --- a/resources/services/directconnect/virtual_interfaces.go +++ b/resources/services/directconnect/virtual_interfaces.go @@ -3,13 +3,14 @@ package directconnect import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/directconnect" "github.com/aws/aws-sdk-go-v2/service/directconnect/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func VirtualInterfaces() *schema.Table { diff --git a/resources/services/directconnect/virtual_interfaces_mock_test.go b/resources/services/directconnect/virtual_interfaces_mock_test.go index 5d270e64..b3b691a4 100644 --- a/resources/services/directconnect/virtual_interfaces_mock_test.go +++ b/resources/services/directconnect/virtual_interfaces_mock_test.go @@ -3,13 +3,14 @@ package directconnect import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/directconnect" "github.com/aws/aws-sdk-go-v2/service/directconnect/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDirectconnectVirtualInterfacesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/dms/replication_instances.go b/resources/services/dms/replication_instances.go index 0891c9e2..52d05694 100644 --- a/resources/services/dms/replication_instances.go +++ b/resources/services/dms/replication_instances.go @@ -3,12 +3,13 @@ package dms import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/dms/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/databasemigrationservice" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/dms/models" ) func ReplicationInstances() *schema.Table { diff --git a/resources/services/dms/replication_instances_mock_test.go b/resources/services/dms/replication_instances_mock_test.go index a86a6907..5b54223d 100644 --- a/resources/services/dms/replication_instances_mock_test.go +++ b/resources/services/dms/replication_instances_mock_test.go @@ -3,14 +3,15 @@ package dms import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/databasemigrationservice" "github.com/aws/aws-sdk-go-v2/service/databasemigrationservice/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDmsReplicationInstances(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/dms/tags.go b/resources/services/dms/tags.go index 7d1fe1c8..d2b18b76 100644 --- a/resources/services/dms/tags.go +++ b/resources/services/dms/tags.go @@ -3,9 +3,10 @@ package dms import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client/services" "github.com/aws/aws-sdk-go-v2/service/databasemigrationservice" "github.com/thoas/go-funk" + + "github.com/OpsHelmInc/cloudquery/v2/client/services" ) // getTags fetches tags for the given resources and returns them in a map. arnPtrPath is the path to the ARN field in the resource. The returned map is keyed by ARN. diff --git a/resources/services/dms/tags_test.go b/resources/services/dms/tags_test.go index 547ca0af..56db9c00 100644 --- a/resources/services/dms/tags_test.go +++ b/resources/services/dms/tags_test.go @@ -4,13 +4,14 @@ import ( "context" "testing" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/databasemigrationservice" "github.com/aws/aws-sdk-go-v2/service/databasemigrationservice/types" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" "github.com/thoas/go-funk" + + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" ) func TestGetTags(t *testing.T) { diff --git a/resources/services/docdb/certificates.go b/resources/services/docdb/certificates.go index ba975957..ed5b8c12 100644 --- a/resources/services/docdb/certificates.go +++ b/resources/services/docdb/certificates.go @@ -3,12 +3,13 @@ package docdb import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/docdb" "github.com/aws/aws-sdk-go-v2/service/docdb/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Certificates() *schema.Table { diff --git a/resources/services/docdb/certificates_mock_test.go b/resources/services/docdb/certificates_mock_test.go index a6822d5c..643ccaba 100644 --- a/resources/services/docdb/certificates_mock_test.go +++ b/resources/services/docdb/certificates_mock_test.go @@ -3,12 +3,13 @@ package docdb import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/docdb" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildCertificatesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/docdb/cluster_parameter_groups.go b/resources/services/docdb/cluster_parameter_groups.go index 1d1350ed..1bf0eb63 100644 --- a/resources/services/docdb/cluster_parameter_groups.go +++ b/resources/services/docdb/cluster_parameter_groups.go @@ -3,13 +3,14 @@ package docdb import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/docdb" "github.com/aws/aws-sdk-go-v2/service/docdb/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func ClusterParameterGroups() *schema.Table { diff --git a/resources/services/docdb/cluster_parameter_groups_mock_test.go b/resources/services/docdb/cluster_parameter_groups_mock_test.go index fd58b640..685cebde 100644 --- a/resources/services/docdb/cluster_parameter_groups_mock_test.go +++ b/resources/services/docdb/cluster_parameter_groups_mock_test.go @@ -3,12 +3,13 @@ package docdb import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/docdb" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildClusterParameterGroupsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/docdb/cluster_snapshots.go b/resources/services/docdb/cluster_snapshots.go index 27fe41e5..a05c6bed 100644 --- a/resources/services/docdb/cluster_snapshots.go +++ b/resources/services/docdb/cluster_snapshots.go @@ -3,13 +3,14 @@ package docdb import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/docdb" "github.com/aws/aws-sdk-go-v2/service/docdb/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func clusterSnapshots() *schema.Table { diff --git a/resources/services/docdb/clusters.go b/resources/services/docdb/clusters.go index 3f65dee8..261a8fca 100644 --- a/resources/services/docdb/clusters.go +++ b/resources/services/docdb/clusters.go @@ -3,14 +3,15 @@ package docdb import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/docdb" "github.com/aws/aws-sdk-go-v2/service/docdb/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Clusters() *schema.Table { diff --git a/resources/services/docdb/clusters_mock_test.go b/resources/services/docdb/clusters_mock_test.go index 23f27fcd..a30b8574 100644 --- a/resources/services/docdb/clusters_mock_test.go +++ b/resources/services/docdb/clusters_mock_test.go @@ -3,12 +3,13 @@ package docdb import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/docdb" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildClustersMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/docdb/event_subscriptions.go b/resources/services/docdb/event_subscriptions.go index 36bea7ab..9f1507a1 100644 --- a/resources/services/docdb/event_subscriptions.go +++ b/resources/services/docdb/event_subscriptions.go @@ -3,13 +3,14 @@ package docdb import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/docdb" "github.com/aws/aws-sdk-go-v2/service/docdb/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func EventSubscriptions() *schema.Table { diff --git a/resources/services/docdb/event_subscriptions_mock_test.go b/resources/services/docdb/event_subscriptions_mock_test.go index 2ef38d04..ceba1585 100644 --- a/resources/services/docdb/event_subscriptions_mock_test.go +++ b/resources/services/docdb/event_subscriptions_mock_test.go @@ -3,12 +3,13 @@ package docdb import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/docdb" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEventSubscriptionsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/docdb/global_clusters.go b/resources/services/docdb/global_clusters.go index 046afe7a..c4450cf9 100644 --- a/resources/services/docdb/global_clusters.go +++ b/resources/services/docdb/global_clusters.go @@ -3,13 +3,14 @@ package docdb import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/docdb" "github.com/aws/aws-sdk-go-v2/service/docdb/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func GlobalClusters() *schema.Table { diff --git a/resources/services/docdb/helpers.go b/resources/services/docdb/helpers.go index d16a3ae0..d588fce2 100644 --- a/resources/services/docdb/helpers.go +++ b/resources/services/docdb/helpers.go @@ -3,10 +3,11 @@ package docdb import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/docdb" - "github.com/cloudquery/plugin-sdk/v4/schema" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" ) func resolveDocDBTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, name, columnName string) error { diff --git a/resources/services/docdb/instances.go b/resources/services/docdb/instances.go index 76a11e8e..556a7cae 100644 --- a/resources/services/docdb/instances.go +++ b/resources/services/docdb/instances.go @@ -3,14 +3,15 @@ package docdb import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/docdb" "github.com/aws/aws-sdk-go-v2/service/docdb/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func instances() *schema.Table { diff --git a/resources/services/docdb/subnet_groups.go b/resources/services/docdb/subnet_groups.go index ba096eb1..72ed22ef 100644 --- a/resources/services/docdb/subnet_groups.go +++ b/resources/services/docdb/subnet_groups.go @@ -3,13 +3,14 @@ package docdb import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/docdb" "github.com/aws/aws-sdk-go-v2/service/docdb/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func SubnetGroups() *schema.Table { diff --git a/resources/services/docdb/subnet_groups_mock_test.go b/resources/services/docdb/subnet_groups_mock_test.go index ab1e90eb..b1992d18 100644 --- a/resources/services/docdb/subnet_groups_mock_test.go +++ b/resources/services/docdb/subnet_groups_mock_test.go @@ -3,12 +3,13 @@ package docdb import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/docdb" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildSubnetGroupsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/dynamodb/backups.go b/resources/services/dynamodb/backups.go index 6d964d1d..8e047bce 100644 --- a/resources/services/dynamodb/backups.go +++ b/resources/services/dynamodb/backups.go @@ -3,13 +3,14 @@ package dynamodb import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/dynamodb" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Backups() *schema.Table { diff --git a/resources/services/dynamodb/backups_mock_test.go b/resources/services/dynamodb/backups_mock_test.go index 792cd1bd..6d3e6049 100644 --- a/resources/services/dynamodb/backups_mock_test.go +++ b/resources/services/dynamodb/backups_mock_test.go @@ -3,13 +3,14 @@ package dynamodb import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/dynamodb" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDynamodbBackupMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/dynamodb/export_mock_test.go b/resources/services/dynamodb/export_mock_test.go index 1fb2198f..1c83896e 100644 --- a/resources/services/dynamodb/export_mock_test.go +++ b/resources/services/dynamodb/export_mock_test.go @@ -3,13 +3,14 @@ package dynamodb import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/dynamodb" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDynamodbExportsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/dynamodb/exports.go b/resources/services/dynamodb/exports.go index 181391e8..86b2291a 100644 --- a/resources/services/dynamodb/exports.go +++ b/resources/services/dynamodb/exports.go @@ -3,12 +3,13 @@ package dynamodb import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/dynamodb" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Exports() *schema.Table { diff --git a/resources/services/dynamodb/global_tables.go b/resources/services/dynamodb/global_tables.go index a9cca7d8..97d61242 100644 --- a/resources/services/dynamodb/global_tables.go +++ b/resources/services/dynamodb/global_tables.go @@ -3,13 +3,14 @@ package dynamodb import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/dynamodb" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func GlobalTables() *schema.Table { diff --git a/resources/services/dynamodb/global_tables_mock_test.go b/resources/services/dynamodb/global_tables_mock_test.go index 35887588..0f41eb58 100644 --- a/resources/services/dynamodb/global_tables_mock_test.go +++ b/resources/services/dynamodb/global_tables_mock_test.go @@ -3,13 +3,14 @@ package dynamodb import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/dynamodb" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDynamodbGlobalTablesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/dynamodb/table_continuous_backups.go b/resources/services/dynamodb/table_continuous_backups.go index 0198a4c0..0dbda7d6 100644 --- a/resources/services/dynamodb/table_continuous_backups.go +++ b/resources/services/dynamodb/table_continuous_backups.go @@ -3,12 +3,13 @@ package dynamodb import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/dynamodb" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func tableContinuousBackups() *schema.Table { diff --git a/resources/services/dynamodb/table_replica_auto_scalings.go b/resources/services/dynamodb/table_replica_auto_scalings.go index 7392fb63..7ddba449 100644 --- a/resources/services/dynamodb/table_replica_auto_scalings.go +++ b/resources/services/dynamodb/table_replica_auto_scalings.go @@ -3,13 +3,14 @@ package dynamodb import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/dynamodb" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func tableReplicaAutoScalings() *schema.Table { diff --git a/resources/services/dynamodb/tables.go b/resources/services/dynamodb/tables.go index 7b416bca..6d79a278 100644 --- a/resources/services/dynamodb/tables.go +++ b/resources/services/dynamodb/tables.go @@ -3,14 +3,15 @@ package dynamodb import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/dynamodb" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Tables() *schema.Table { diff --git a/resources/services/dynamodb/tables_mock_test.go b/resources/services/dynamodb/tables_mock_test.go index 1aa73cd5..048cd44e 100644 --- a/resources/services/dynamodb/tables_mock_test.go +++ b/resources/services/dynamodb/tables_mock_test.go @@ -3,14 +3,15 @@ package dynamodb import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/dynamodb" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDynamodbTablesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/dynamodbstreams/streams.go b/resources/services/dynamodbstreams/streams.go index af6fb208..c2cb05d8 100644 --- a/resources/services/dynamodbstreams/streams.go +++ b/resources/services/dynamodbstreams/streams.go @@ -3,13 +3,14 @@ package dynamodbstreams import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/dynamodbstreams" "github.com/aws/aws-sdk-go-v2/service/dynamodbstreams/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Streams() *schema.Table { diff --git a/resources/services/dynamodbstreams/streams_mock_test.go b/resources/services/dynamodbstreams/streams_mock_test.go index ed5fb387..f5e526ce 100644 --- a/resources/services/dynamodbstreams/streams_mock_test.go +++ b/resources/services/dynamodbstreams/streams_mock_test.go @@ -3,13 +3,14 @@ package dynamodbstreams import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/dynamodbstreams" "github.com/aws/aws-sdk-go-v2/service/dynamodbstreams/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDynamodbstreamsStreamsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/capacity_reservations.go b/resources/services/ec2/capacity_reservations.go index 02edb4c1..1d678e44 100644 --- a/resources/services/ec2/capacity_reservations.go +++ b/resources/services/ec2/capacity_reservations.go @@ -3,13 +3,14 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func CapacityReservations() *schema.Table { diff --git a/resources/services/ec2/capacity_reservations_mock_test.go b/resources/services/ec2/capacity_reservations_mock_test.go index e73750a2..dfe5b4a6 100644 --- a/resources/services/ec2/capacity_reservations_mock_test.go +++ b/resources/services/ec2/capacity_reservations_mock_test.go @@ -3,13 +3,14 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEc2CapacityReservations(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/customer_gateways.go b/resources/services/ec2/customer_gateways.go index 6f13f713..0e21420e 100644 --- a/resources/services/ec2/customer_gateways.go +++ b/resources/services/ec2/customer_gateways.go @@ -3,15 +3,16 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func CustomerGateways() *schema.Table { diff --git a/resources/services/ec2/customer_gateways_mock_test.go b/resources/services/ec2/customer_gateways_mock_test.go index ef0ddba0..1cc42004 100644 --- a/resources/services/ec2/customer_gateways_mock_test.go +++ b/resources/services/ec2/customer_gateways_mock_test.go @@ -3,13 +3,14 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEc2CustomerGateways(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/ebs_snapshot_attributes.go b/resources/services/ec2/ebs_snapshot_attributes.go index 0a534974..8c0ef3ea 100644 --- a/resources/services/ec2/ebs_snapshot_attributes.go +++ b/resources/services/ec2/ebs_snapshot_attributes.go @@ -3,13 +3,14 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ebsSnapshotAttributes() *schema.Table { diff --git a/resources/services/ec2/ebs_snapshots.go b/resources/services/ec2/ebs_snapshots.go index 7bb1ec17..972be821 100644 --- a/resources/services/ec2/ebs_snapshots.go +++ b/resources/services/ec2/ebs_snapshots.go @@ -3,15 +3,16 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func EbsSnapshots() *schema.Table { diff --git a/resources/services/ec2/ebs_snapshots_mock_test.go b/resources/services/ec2/ebs_snapshots_mock_test.go index 81a6155c..62a6cd51 100644 --- a/resources/services/ec2/ebs_snapshots_mock_test.go +++ b/resources/services/ec2/ebs_snapshots_mock_test.go @@ -3,14 +3,15 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEc2EbsSnapshots(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/ebs_volumes.go b/resources/services/ec2/ebs_volumes.go index 15b03e90..1b4670ad 100644 --- a/resources/services/ec2/ebs_volumes.go +++ b/resources/services/ec2/ebs_volumes.go @@ -3,15 +3,16 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func EbsVolumes() *schema.Table { diff --git a/resources/services/ec2/ebs_volumes_mock_test.go b/resources/services/ec2/ebs_volumes_mock_test.go index 021a83d7..532f0ba2 100644 --- a/resources/services/ec2/ebs_volumes_mock_test.go +++ b/resources/services/ec2/ebs_volumes_mock_test.go @@ -3,12 +3,13 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ec2" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEc2EbsVolumes(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/egress_only_internet_gateways.go b/resources/services/ec2/egress_only_internet_gateways.go index 0b82d32e..4f949578 100644 --- a/resources/services/ec2/egress_only_internet_gateways.go +++ b/resources/services/ec2/egress_only_internet_gateways.go @@ -3,15 +3,16 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func EgressOnlyInternetGateways() *schema.Table { diff --git a/resources/services/ec2/egress_only_internet_gateways_mock_test.go b/resources/services/ec2/egress_only_internet_gateways_mock_test.go index 025d2ac1..dfe41c41 100644 --- a/resources/services/ec2/egress_only_internet_gateways_mock_test.go +++ b/resources/services/ec2/egress_only_internet_gateways_mock_test.go @@ -3,12 +3,13 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ec2" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEgressOnlyInternetGateways(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/eips.go b/resources/services/ec2/eips.go index af56f8f8..7b42ddef 100644 --- a/resources/services/ec2/eips.go +++ b/resources/services/ec2/eips.go @@ -3,13 +3,14 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Eips() *schema.Table { diff --git a/resources/services/ec2/flow_logs.go b/resources/services/ec2/flow_logs.go index cffe2cb3..88772bff 100644 --- a/resources/services/ec2/flow_logs.go +++ b/resources/services/ec2/flow_logs.go @@ -3,15 +3,16 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func FlowLogs() *schema.Table { diff --git a/resources/services/ec2/flow_logs_mock_test.go b/resources/services/ec2/flow_logs_mock_test.go index a61b50af..79a0daed 100644 --- a/resources/services/ec2/flow_logs_mock_test.go +++ b/resources/services/ec2/flow_logs_mock_test.go @@ -3,13 +3,14 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEc2FlowLogsMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -26,6 +27,7 @@ func buildEc2FlowLogsMock(t *testing.T, ctrl *gomock.Controller) client.Services Ec2: m, } } + func TestEc2FlowLogs(t *testing.T) { client.AwsMockTestHelper(t, FlowLogs(), buildEc2FlowLogsMock, client.TestOptions{}) } diff --git a/resources/services/ec2/hosts.go b/resources/services/ec2/hosts.go index 481fd7c9..8dbb823a 100644 --- a/resources/services/ec2/hosts.go +++ b/resources/services/ec2/hosts.go @@ -3,15 +3,16 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Hosts() *schema.Table { diff --git a/resources/services/ec2/hosts_mock_test.go b/resources/services/ec2/hosts_mock_test.go index a470f8b3..d09a0b8d 100644 --- a/resources/services/ec2/hosts_mock_test.go +++ b/resources/services/ec2/hosts_mock_test.go @@ -3,13 +3,14 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEc2Hosts(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/image_attributes.go b/resources/services/ec2/image_attributes.go index b451c6f7..16ca0040 100644 --- a/resources/services/ec2/image_attributes.go +++ b/resources/services/ec2/image_attributes.go @@ -3,13 +3,14 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func imageAttributesLaunchPermissions() *schema.Table { diff --git a/resources/services/ec2/image_last_launched.go b/resources/services/ec2/image_last_launched.go index bf2176d4..8eefef37 100644 --- a/resources/services/ec2/image_last_launched.go +++ b/resources/services/ec2/image_last_launched.go @@ -3,12 +3,13 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" ) func imageAttributesLastLaunchTime() *schema.Table { diff --git a/resources/services/ec2/images.go b/resources/services/ec2/images.go index 2e0e8989..3a61a874 100644 --- a/resources/services/ec2/images.go +++ b/resources/services/ec2/images.go @@ -4,16 +4,17 @@ import ( "context" "reflect" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" "golang.org/x/sync/errgroup" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Images() *schema.Table { diff --git a/resources/services/ec2/images_mock_test.go b/resources/services/ec2/images_mock_test.go index 82092832..b5ee7f1c 100644 --- a/resources/services/ec2/images_mock_test.go +++ b/resources/services/ec2/images_mock_test.go @@ -3,14 +3,15 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEc2ImagesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/instance_connect.go b/resources/services/ec2/instance_connect.go index d8c86424..6b0b8091 100644 --- a/resources/services/ec2/instance_connect.go +++ b/resources/services/ec2/instance_connect.go @@ -3,13 +3,14 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func InstanceConnectEndpoints() *schema.Table { diff --git a/resources/services/ec2/instance_connect_mock_test.go b/resources/services/ec2/instance_connect_mock_test.go index 323502f9..12e33511 100644 --- a/resources/services/ec2/instance_connect_mock_test.go +++ b/resources/services/ec2/instance_connect_mock_test.go @@ -3,14 +3,15 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/aws/smithy-go/middleware" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildInstanceConnectEndpoint(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/instances.go b/resources/services/ec2/instances.go index 3607d3c0..18b2cf8b 100644 --- a/resources/services/ec2/instances.go +++ b/resources/services/ec2/instances.go @@ -5,15 +5,16 @@ import ( "regexp" "time" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Instances() *schema.Table { diff --git a/resources/services/ec2/instances_mock_test.go b/resources/services/ec2/instances_mock_test.go index fa5be442..598aedea 100644 --- a/resources/services/ec2/instances_mock_test.go +++ b/resources/services/ec2/instances_mock_test.go @@ -3,14 +3,15 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEc2Instances(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/internet_gateways.go b/resources/services/ec2/internet_gateways.go index e85e6797..08488862 100644 --- a/resources/services/ec2/internet_gateways.go +++ b/resources/services/ec2/internet_gateways.go @@ -3,15 +3,16 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func InternetGateways() *schema.Table { diff --git a/resources/services/ec2/internet_gateways_mock_test.go b/resources/services/ec2/internet_gateways_mock_test.go index 5872d2a9..3580d3b7 100644 --- a/resources/services/ec2/internet_gateways_mock_test.go +++ b/resources/services/ec2/internet_gateways_mock_test.go @@ -3,13 +3,14 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEc2InternetGateways(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/key_pairs.go b/resources/services/ec2/key_pairs.go index 105cad5f..84458c77 100644 --- a/resources/services/ec2/key_pairs.go +++ b/resources/services/ec2/key_pairs.go @@ -3,15 +3,16 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func KeyPairs() *schema.Table { diff --git a/resources/services/ec2/key_pairs_mock_test.go b/resources/services/ec2/key_pairs_mock_test.go index 8f87fed0..09378cd9 100644 --- a/resources/services/ec2/key_pairs_mock_test.go +++ b/resources/services/ec2/key_pairs_mock_test.go @@ -3,13 +3,14 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEc2KeyPairs(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/launch_template_versions.go b/resources/services/ec2/launch_template_versions.go index bda2a609..d174f5a9 100644 --- a/resources/services/ec2/launch_template_versions.go +++ b/resources/services/ec2/launch_template_versions.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func LaunchTemplateVersions() *schema.Table { diff --git a/resources/services/ec2/launch_template_versions_mock_test.go b/resources/services/ec2/launch_template_versions_mock_test.go index f6d3e35d..3cbf9533 100644 --- a/resources/services/ec2/launch_template_versions_mock_test.go +++ b/resources/services/ec2/launch_template_versions_mock_test.go @@ -3,13 +3,14 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEc2LaunchTemplateVersions(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/launch_templates.go b/resources/services/ec2/launch_templates.go index 1d0b721b..90396cd5 100644 --- a/resources/services/ec2/launch_templates.go +++ b/resources/services/ec2/launch_templates.go @@ -3,15 +3,16 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func LaunchTemplates() *schema.Table { diff --git a/resources/services/ec2/launch_templates_mock_test.go b/resources/services/ec2/launch_templates_mock_test.go index 5061552f..82a9737c 100644 --- a/resources/services/ec2/launch_templates_mock_test.go +++ b/resources/services/ec2/launch_templates_mock_test.go @@ -3,13 +3,14 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEc2LaunchTemplates(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/managed_prefix_lists.go b/resources/services/ec2/managed_prefix_lists.go index ae1305cc..c97971fc 100644 --- a/resources/services/ec2/managed_prefix_lists.go +++ b/resources/services/ec2/managed_prefix_lists.go @@ -3,13 +3,14 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func ManagedPrefixLists() *schema.Table { @@ -39,6 +40,7 @@ The 'request_account_id' and 'request_region' columns are added to show the acco }, } } + func fetchEc2ManagedPrefixLists(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { cl := meta.(*client.Client) svc := cl.Services(client.AWSServiceEc2).Ec2 diff --git a/resources/services/ec2/managed_prefix_lists_mock_test.go b/resources/services/ec2/managed_prefix_lists_mock_test.go index 4cf2b5ca..a8a2ed1b 100644 --- a/resources/services/ec2/managed_prefix_lists_mock_test.go +++ b/resources/services/ec2/managed_prefix_lists_mock_test.go @@ -3,13 +3,14 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEc2ManagedPrefixList(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/nat_gateways.go b/resources/services/ec2/nat_gateways.go index e86e693a..a1f6ba16 100644 --- a/resources/services/ec2/nat_gateways.go +++ b/resources/services/ec2/nat_gateways.go @@ -3,15 +3,16 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func NatGateways() *schema.Table { diff --git a/resources/services/ec2/nat_gateways_mock_test.go b/resources/services/ec2/nat_gateways_mock_test.go index 08230545..0fd98af0 100644 --- a/resources/services/ec2/nat_gateways_mock_test.go +++ b/resources/services/ec2/nat_gateways_mock_test.go @@ -3,13 +3,14 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEc2NatGateways(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/network_acls.go b/resources/services/ec2/network_acls.go index b86039e2..b1f953c4 100644 --- a/resources/services/ec2/network_acls.go +++ b/resources/services/ec2/network_acls.go @@ -3,15 +3,16 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func NetworkAcls() *schema.Table { diff --git a/resources/services/ec2/network_acls_mock_test.go b/resources/services/ec2/network_acls_mock_test.go index df39220b..78ae7521 100644 --- a/resources/services/ec2/network_acls_mock_test.go +++ b/resources/services/ec2/network_acls_mock_test.go @@ -3,14 +3,15 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEc2NetworkAcls(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/network_interfaces.go b/resources/services/ec2/network_interfaces.go index 7ae79581..a22332ee 100644 --- a/resources/services/ec2/network_interfaces.go +++ b/resources/services/ec2/network_interfaces.go @@ -3,15 +3,16 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func NetworkInterfaces() *schema.Table { diff --git a/resources/services/ec2/network_interfaces_mock_test.go b/resources/services/ec2/network_interfaces_mock_test.go index 3f6f589c..692a51b8 100644 --- a/resources/services/ec2/network_interfaces_mock_test.go +++ b/resources/services/ec2/network_interfaces_mock_test.go @@ -3,12 +3,13 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ec2" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEc2NetworkInterfaces(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/regional_config_mock_test.go b/resources/services/ec2/regional_config_mock_test.go index 21c95f6d..5bcec7ec 100644 --- a/resources/services/ec2/regional_config_mock_test.go +++ b/resources/services/ec2/regional_config_mock_test.go @@ -3,11 +3,12 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/golang/mock/gomock" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" ) func buildEc2RegionalConfig(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/regional_configs.go b/resources/services/ec2/regional_configs.go index 5659cd42..ab31acd2 100644 --- a/resources/services/ec2/regional_configs.go +++ b/resources/services/ec2/regional_configs.go @@ -3,13 +3,14 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/ec2/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ec2" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/ec2/models" ) func RegionalConfigs() *schema.Table { diff --git a/resources/services/ec2/regions.go b/resources/services/ec2/regions.go index 7575903b..81ab9e35 100644 --- a/resources/services/ec2/regions.go +++ b/resources/services/ec2/regions.go @@ -3,14 +3,15 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Regions() *schema.Table { diff --git a/resources/services/ec2/regions_mock_test.go b/resources/services/ec2/regions_mock_test.go index 1fdc4c3b..8cac2ed0 100644 --- a/resources/services/ec2/regions_mock_test.go +++ b/resources/services/ec2/regions_mock_test.go @@ -3,14 +3,15 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRegionsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/reserved_instances.go b/resources/services/ec2/reserved_instances.go index af296b21..7660fc8d 100644 --- a/resources/services/ec2/reserved_instances.go +++ b/resources/services/ec2/reserved_instances.go @@ -3,15 +3,16 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func ReservedInstances() *schema.Table { diff --git a/resources/services/ec2/reserved_instances_mock_test.go b/resources/services/ec2/reserved_instances_mock_test.go index 0758f370..f4867bf0 100644 --- a/resources/services/ec2/reserved_instances_mock_test.go +++ b/resources/services/ec2/reserved_instances_mock_test.go @@ -3,13 +3,14 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildReservedEc2Instances(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/route_tables.go b/resources/services/ec2/route_tables.go index 459dde0e..25b1033a 100644 --- a/resources/services/ec2/route_tables.go +++ b/resources/services/ec2/route_tables.go @@ -3,15 +3,16 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func RouteTables() *schema.Table { diff --git a/resources/services/ec2/route_tables_mock_test.go b/resources/services/ec2/route_tables_mock_test.go index 3a12bef2..ed31e555 100644 --- a/resources/services/ec2/route_tables_mock_test.go +++ b/resources/services/ec2/route_tables_mock_test.go @@ -3,13 +3,14 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEc2RouteTables(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/security_groups.go b/resources/services/ec2/security_groups.go index 0531356f..0b4c4dea 100644 --- a/resources/services/ec2/security_groups.go +++ b/resources/services/ec2/security_groups.go @@ -3,15 +3,16 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func SecurityGroups() *schema.Table { diff --git a/resources/services/ec2/security_groups_mock_test.go b/resources/services/ec2/security_groups_mock_test.go index 56fc6095..89dbb5fe 100644 --- a/resources/services/ec2/security_groups_mock_test.go +++ b/resources/services/ec2/security_groups_mock_test.go @@ -3,13 +3,14 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEc2SecurityGroups(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/subnets.go b/resources/services/ec2/subnets.go index 472bdd75..78a297ea 100644 --- a/resources/services/ec2/subnets.go +++ b/resources/services/ec2/subnets.go @@ -3,13 +3,14 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Subnets() *schema.Table { diff --git a/resources/services/ec2/subnets_mock_test.go b/resources/services/ec2/subnets_mock_test.go index ffe02175..e5219901 100644 --- a/resources/services/ec2/subnets_mock_test.go +++ b/resources/services/ec2/subnets_mock_test.go @@ -3,13 +3,14 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEc2Subnets(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/transit_gateway_attachments.go b/resources/services/ec2/transit_gateway_attachments.go index 394d2ae2..e07ba50b 100644 --- a/resources/services/ec2/transit_gateway_attachments.go +++ b/resources/services/ec2/transit_gateway_attachments.go @@ -3,13 +3,14 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func transitGatewayAttachments() *schema.Table { diff --git a/resources/services/ec2/transit_gateway_multicast_domains.go b/resources/services/ec2/transit_gateway_multicast_domains.go index 82644c7d..e62713a8 100644 --- a/resources/services/ec2/transit_gateway_multicast_domains.go +++ b/resources/services/ec2/transit_gateway_multicast_domains.go @@ -3,13 +3,14 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func transitGatewayMulticastDomains() *schema.Table { diff --git a/resources/services/ec2/transit_gateway_peering_attachments.go b/resources/services/ec2/transit_gateway_peering_attachments.go index 667b50bb..bdae11ce 100644 --- a/resources/services/ec2/transit_gateway_peering_attachments.go +++ b/resources/services/ec2/transit_gateway_peering_attachments.go @@ -3,13 +3,14 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func transitGatewayPeeringAttachments() *schema.Table { diff --git a/resources/services/ec2/transit_gateway_route_tables.go b/resources/services/ec2/transit_gateway_route_tables.go index fba47adc..fdcc48d0 100644 --- a/resources/services/ec2/transit_gateway_route_tables.go +++ b/resources/services/ec2/transit_gateway_route_tables.go @@ -3,13 +3,14 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func transitGatewayRouteTables() *schema.Table { diff --git a/resources/services/ec2/transit_gateway_vpc_attachments.go b/resources/services/ec2/transit_gateway_vpc_attachments.go index 12e2945c..862047bb 100644 --- a/resources/services/ec2/transit_gateway_vpc_attachments.go +++ b/resources/services/ec2/transit_gateway_vpc_attachments.go @@ -3,13 +3,14 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func transitGatewayVpcAttachments() *schema.Table { diff --git a/resources/services/ec2/transit_gateways.go b/resources/services/ec2/transit_gateways.go index d8c1d3b9..af53096b 100644 --- a/resources/services/ec2/transit_gateways.go +++ b/resources/services/ec2/transit_gateways.go @@ -3,12 +3,13 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func TransitGateways() *schema.Table { diff --git a/resources/services/ec2/transit_gateways_mock_test.go b/resources/services/ec2/transit_gateways_mock_test.go index 1e4b9d99..418e3116 100644 --- a/resources/services/ec2/transit_gateways_mock_test.go +++ b/resources/services/ec2/transit_gateways_mock_test.go @@ -3,13 +3,14 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEc2TransitGateways(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/vpc_endpoint_service_permissions.go b/resources/services/ec2/vpc_endpoint_service_permissions.go index 959f845a..9cd5c0bd 100644 --- a/resources/services/ec2/vpc_endpoint_service_permissions.go +++ b/resources/services/ec2/vpc_endpoint_service_permissions.go @@ -3,13 +3,14 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func vpcEndpointServicePermissions() *schema.Table { @@ -30,6 +31,7 @@ func vpcEndpointServicePermissions() *schema.Table { }, } } + func fetchEc2VpcEndpointServicePermissions(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { endpointService := parent.Item.(types.ServiceDetail) if aws.ToString(endpointService.Owner) == "amazon" { diff --git a/resources/services/ec2/vpc_endpoint_services.go b/resources/services/ec2/vpc_endpoint_services.go index 0e08b9e8..ba117738 100644 --- a/resources/services/ec2/vpc_endpoint_services.go +++ b/resources/services/ec2/vpc_endpoint_services.go @@ -3,13 +3,14 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func VpcEndpointServices() *schema.Table { @@ -34,6 +35,7 @@ func VpcEndpointServices() *schema.Table { }, } } + func fetchEc2VpcEndpointServices(ctx context.Context, meta schema.ClientMeta, _ *schema.Resource, res chan<- any) error { var config ec2.DescribeVpcEndpointServicesInput cl := meta.(*client.Client) diff --git a/resources/services/ec2/vpc_endpoint_services_mock_test.go b/resources/services/ec2/vpc_endpoint_services_mock_test.go index 1da19de9..fc22b6b2 100644 --- a/resources/services/ec2/vpc_endpoint_services_mock_test.go +++ b/resources/services/ec2/vpc_endpoint_services_mock_test.go @@ -3,13 +3,14 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEc2VpcEndpointServices(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/vpc_endpoints.go b/resources/services/ec2/vpc_endpoints.go index c27b7853..fb23e661 100644 --- a/resources/services/ec2/vpc_endpoints.go +++ b/resources/services/ec2/vpc_endpoints.go @@ -3,15 +3,16 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func VpcEndpoints() *schema.Table { @@ -40,6 +41,7 @@ func VpcEndpoints() *schema.Table { }, } } + func fetchEc2VpcEndpoints(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { cl := meta.(*client.Client) svc := cl.Services(client.AWSServiceEc2).Ec2 diff --git a/resources/services/ec2/vpc_endpoints_mock_test.go b/resources/services/ec2/vpc_endpoints_mock_test.go index b3a8c742..02576f41 100644 --- a/resources/services/ec2/vpc_endpoints_mock_test.go +++ b/resources/services/ec2/vpc_endpoints_mock_test.go @@ -3,13 +3,14 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEc2VpcEndpoints(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/vpc_peering_connections.go b/resources/services/ec2/vpc_peering_connections.go index 872d8f6b..d10e5b49 100644 --- a/resources/services/ec2/vpc_peering_connections.go +++ b/resources/services/ec2/vpc_peering_connections.go @@ -3,15 +3,16 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func VpcPeeringConnections() *schema.Table { diff --git a/resources/services/ec2/vpcs.go b/resources/services/ec2/vpcs.go index aea03e9a..737138bc 100644 --- a/resources/services/ec2/vpcs.go +++ b/resources/services/ec2/vpcs.go @@ -3,15 +3,16 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Vpcs() *schema.Table { diff --git a/resources/services/ec2/vpcs_mock_test.go b/resources/services/ec2/vpcs_mock_test.go index 5093f8fe..6501c2ca 100644 --- a/resources/services/ec2/vpcs_mock_test.go +++ b/resources/services/ec2/vpcs_mock_test.go @@ -3,13 +3,14 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEc2Vpcs(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -25,6 +26,7 @@ func buildEc2Vpcs(t *testing.T, ctrl *gomock.Controller) client.Services { Ec2: m, } } + func TestEc2Vpcs(t *testing.T) { client.AwsMockTestHelper(t, Vpcs(), buildEc2Vpcs, client.TestOptions{}) } diff --git a/resources/services/ec2/vpcs_peering_connections_mock_test.go b/resources/services/ec2/vpcs_peering_connections_mock_test.go index 3ef45da8..bb1d6cd5 100644 --- a/resources/services/ec2/vpcs_peering_connections_mock_test.go +++ b/resources/services/ec2/vpcs_peering_connections_mock_test.go @@ -3,13 +3,14 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEc2VpcsPeeringConnections(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/vpn_connections.go b/resources/services/ec2/vpn_connections.go index c8456581..0fbfd26b 100644 --- a/resources/services/ec2/vpn_connections.go +++ b/resources/services/ec2/vpn_connections.go @@ -3,12 +3,13 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func VpnConnections() *schema.Table { diff --git a/resources/services/ec2/vpn_connections_mock_test.go b/resources/services/ec2/vpn_connections_mock_test.go index e912f59f..e6ecd0ce 100644 --- a/resources/services/ec2/vpn_connections_mock_test.go +++ b/resources/services/ec2/vpn_connections_mock_test.go @@ -3,13 +3,14 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildVpnConnections(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ec2/vpn_gateways.go b/resources/services/ec2/vpn_gateways.go index eb224195..fb13fced 100644 --- a/resources/services/ec2/vpn_gateways.go +++ b/resources/services/ec2/vpn_gateways.go @@ -3,15 +3,16 @@ package ec2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func VpnGateways() *schema.Table { diff --git a/resources/services/ec2/vpn_gateways_mock_test.go b/resources/services/ec2/vpn_gateways_mock_test.go index f7d648e4..ed7211ca 100644 --- a/resources/services/ec2/vpn_gateways_mock_test.go +++ b/resources/services/ec2/vpn_gateways_mock_test.go @@ -3,13 +3,14 @@ package ec2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEc2VpnGateways(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ecr/lifecycle_policy.go b/resources/services/ecr/lifecycle_policy.go index e16779d0..80db666b 100644 --- a/resources/services/ecr/lifecycle_policy.go +++ b/resources/services/ecr/lifecycle_policy.go @@ -3,13 +3,14 @@ package ecr import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/ecr" "github.com/aws/aws-sdk-go-v2/service/ecr/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func lifeCyclePolicy() *schema.Table { @@ -38,6 +39,7 @@ func lifeCyclePolicy() *schema.Table { }, } } + func fetchRepositoryLifecyclePolicy(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { cl := meta.(*client.Client) svc := cl.Services(client.AWSServiceEcr).Ecr diff --git a/resources/services/ecr/repositories.go b/resources/services/ecr/repositories.go index 760155d0..fde0b557 100644 --- a/resources/services/ecr/repositories.go +++ b/resources/services/ecr/repositories.go @@ -3,14 +3,15 @@ package ecr import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ecr" "github.com/aws/aws-sdk-go-v2/service/ecr/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Repositories() *schema.Table { @@ -45,6 +46,7 @@ func Repositories() *schema.Table { }, } } + func fetchEcrRepositories(ctx context.Context, meta schema.ClientMeta, _ *schema.Resource, res chan<- any) error { cl := meta.(*client.Client) svc := cl.Services(client.AWSServiceEcr).Ecr diff --git a/resources/services/ecr/repositories_mock_test.go b/resources/services/ecr/repositories_mock_test.go index a03b9ad8..12d5340f 100644 --- a/resources/services/ecr/repositories_mock_test.go +++ b/resources/services/ecr/repositories_mock_test.go @@ -3,13 +3,14 @@ package ecr import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ecr" "github.com/aws/aws-sdk-go-v2/service/ecr/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEcrRepositoriesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ecr/repository_image_scan_findings.go b/resources/services/ecr/repository_image_scan_findings.go index e6db694b..0c69d12e 100644 --- a/resources/services/ecr/repository_image_scan_findings.go +++ b/resources/services/ecr/repository_image_scan_findings.go @@ -3,13 +3,14 @@ package ecr import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ecr" "github.com/aws/aws-sdk-go-v2/service/ecr/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func repositoryImageScanFindings() *schema.Table { @@ -39,6 +40,7 @@ func repositoryImageScanFindings() *schema.Table { }, } } + func fetchEcrRepositoryImageScanFindings(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { cl := meta.(*client.Client) svc := cl.Services(client.AWSServiceEcr).Ecr diff --git a/resources/services/ecr/repository_images.go b/resources/services/ecr/repository_images.go index d4cd80fe..77533a64 100644 --- a/resources/services/ecr/repository_images.go +++ b/resources/services/ecr/repository_images.go @@ -3,13 +3,14 @@ package ecr import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ecr" "github.com/aws/aws-sdk-go-v2/service/ecr/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func repositoryImages() *schema.Table { @@ -34,6 +35,7 @@ func repositoryImages() *schema.Table { Relations: schema.Tables{repositoryImageScanFindings()}, } } + func fetchEcrRepositoryImages(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { cl := meta.(*client.Client) svc := cl.Services(client.AWSServiceEcr).Ecr diff --git a/resources/services/ecr/repository_policy.go b/resources/services/ecr/repository_policy.go index 4ecb1651..d8f77a03 100644 --- a/resources/services/ecr/repository_policy.go +++ b/resources/services/ecr/repository_policy.go @@ -3,13 +3,14 @@ package ecr import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/ecr" "github.com/aws/aws-sdk-go-v2/service/ecr/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func repositoryPolicy() *schema.Table { @@ -38,6 +39,7 @@ func repositoryPolicy() *schema.Table { }, } } + func fetchRepositoryPolicy(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { cl := meta.(*client.Client) svc := cl.Services(client.AWSServiceEcr).Ecr diff --git a/resources/services/ecrpublic/repositories.go b/resources/services/ecrpublic/repositories.go index 9036f1bd..2cfc852b 100644 --- a/resources/services/ecrpublic/repositories.go +++ b/resources/services/ecrpublic/repositories.go @@ -3,14 +3,15 @@ package ecrpublic import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ecrpublic" "github.com/aws/aws-sdk-go-v2/service/ecrpublic/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Repositories() *schema.Table { diff --git a/resources/services/ecrpublic/repositories_mock_test.go b/resources/services/ecrpublic/repositories_mock_test.go index bdc14684..6bf06e54 100644 --- a/resources/services/ecrpublic/repositories_mock_test.go +++ b/resources/services/ecrpublic/repositories_mock_test.go @@ -3,13 +3,14 @@ package ecrpublic import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ecrpublic" "github.com/aws/aws-sdk-go-v2/service/ecrpublic/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEcrPublicRepositoriesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ecrpublic/repository_images.go b/resources/services/ecrpublic/repository_images.go index 12f7fc35..5643bb62 100644 --- a/resources/services/ecrpublic/repository_images.go +++ b/resources/services/ecrpublic/repository_images.go @@ -3,14 +3,15 @@ package ecrpublic import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ecrpublic" "github.com/aws/aws-sdk-go-v2/service/ecrpublic/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func repositoryImages() *schema.Table { diff --git a/resources/services/ecs/cluster_container_instances.go b/resources/services/ecs/cluster_container_instances.go index 93232e9a..b718f1d2 100644 --- a/resources/services/ecs/cluster_container_instances.go +++ b/resources/services/ecs/cluster_container_instances.go @@ -3,13 +3,14 @@ package ecs import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/ecs" "github.com/aws/aws-sdk-go-v2/service/ecs/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func clusterContainerInstances() *schema.Table { diff --git a/resources/services/ecs/cluster_mock_test.go b/resources/services/ecs/cluster_mock_test.go index 55eeab48..7bd62e8d 100644 --- a/resources/services/ecs/cluster_mock_test.go +++ b/resources/services/ecs/cluster_mock_test.go @@ -3,13 +3,14 @@ package ecs import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ecs" "github.com/aws/aws-sdk-go-v2/service/ecs/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEcsClusterMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ecs/cluster_services.go b/resources/services/ecs/cluster_services.go index cb172de2..e1292550 100644 --- a/resources/services/ecs/cluster_services.go +++ b/resources/services/ecs/cluster_services.go @@ -3,13 +3,14 @@ package ecs import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/ecs" "github.com/aws/aws-sdk-go-v2/service/ecs/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func clusterServices() *schema.Table { diff --git a/resources/services/ecs/cluster_task_sets.go b/resources/services/ecs/cluster_task_sets.go index eba543a0..49a06928 100644 --- a/resources/services/ecs/cluster_task_sets.go +++ b/resources/services/ecs/cluster_task_sets.go @@ -3,13 +3,14 @@ package ecs import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/ecs" "github.com/aws/aws-sdk-go-v2/service/ecs/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func clusterTaskSets() *schema.Table { @@ -37,6 +38,7 @@ func clusterTaskSets() *schema.Table { }, } } + func fetchEcsClusterTaskSets(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, res chan<- any) error { cluster := resource.Parent.Item.(types.Cluster) service := resource.Item.(types.Service) diff --git a/resources/services/ecs/cluster_tasks.go b/resources/services/ecs/cluster_tasks.go index a89084fd..674447bf 100644 --- a/resources/services/ecs/cluster_tasks.go +++ b/resources/services/ecs/cluster_tasks.go @@ -3,14 +3,15 @@ package ecs import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ecs" "github.com/aws/aws-sdk-go-v2/service/ecs/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func clusterTasks() *schema.Table { diff --git a/resources/services/ecs/clusters.go b/resources/services/ecs/clusters.go index a5825b50..e4927f65 100644 --- a/resources/services/ecs/clusters.go +++ b/resources/services/ecs/clusters.go @@ -3,13 +3,14 @@ package ecs import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/ecs" "github.com/aws/aws-sdk-go-v2/service/ecs/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Clusters() *schema.Table { diff --git a/resources/services/ecs/task_definitions.go b/resources/services/ecs/task_definitions.go index 81a99f41..c37c3aa6 100644 --- a/resources/services/ecs/task_definitions.go +++ b/resources/services/ecs/task_definitions.go @@ -4,15 +4,16 @@ import ( "context" "errors" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/ecs/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ecs" "github.com/aws/aws-sdk-go-v2/service/ecs/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/ecs/models" ) func TaskDefinitions() *schema.Table { diff --git a/resources/services/ecs/task_definitions_mock_test.go b/resources/services/ecs/task_definitions_mock_test.go index 97ca1eea..9592f26d 100644 --- a/resources/services/ecs/task_definitions_mock_test.go +++ b/resources/services/ecs/task_definitions_mock_test.go @@ -3,12 +3,13 @@ package ecs import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ecs" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEcsTaskDefinitions(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/efs/accesspoints.go b/resources/services/efs/accesspoints.go index bdd817b6..4ff90785 100644 --- a/resources/services/efs/accesspoints.go +++ b/resources/services/efs/accesspoints.go @@ -3,13 +3,14 @@ package efs import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/efs" "github.com/aws/aws-sdk-go-v2/service/efs/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - cqtypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + cqtypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func AccessPoints() *schema.Table { diff --git a/resources/services/efs/accesspoints_mock_test.go b/resources/services/efs/accesspoints_mock_test.go index 0d40b496..878459a5 100644 --- a/resources/services/efs/accesspoints_mock_test.go +++ b/resources/services/efs/accesspoints_mock_test.go @@ -3,13 +3,14 @@ package efs import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/efs" "github.com/aws/aws-sdk-go-v2/service/efs/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEfsAccessPointsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/efs/filesystems.go b/resources/services/efs/filesystems.go index a68ba356..28bdc872 100644 --- a/resources/services/efs/filesystems.go +++ b/resources/services/efs/filesystems.go @@ -3,13 +3,14 @@ package efs import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/efs" "github.com/aws/aws-sdk-go-v2/service/efs/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Filesystems() *schema.Table { diff --git a/resources/services/efs/filesystems_mock_test.go b/resources/services/efs/filesystems_mock_test.go index e802bca6..01b66d72 100644 --- a/resources/services/efs/filesystems_mock_test.go +++ b/resources/services/efs/filesystems_mock_test.go @@ -3,13 +3,14 @@ package efs import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/efs" "github.com/aws/aws-sdk-go-v2/service/efs/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEfsFilesystemsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/eks/add_ons.go b/resources/services/eks/add_ons.go index df8d4ef8..bc2dd6e9 100644 --- a/resources/services/eks/add_ons.go +++ b/resources/services/eks/add_ons.go @@ -3,12 +3,13 @@ package eks import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/eks" "github.com/aws/aws-sdk-go-v2/service/eks/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func addOns() *schema.Table { diff --git a/resources/services/eks/clusters.go b/resources/services/eks/clusters.go index 438bc826..af94aa34 100644 --- a/resources/services/eks/clusters.go +++ b/resources/services/eks/clusters.go @@ -3,12 +3,13 @@ package eks import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/eks" "github.com/aws/aws-sdk-go-v2/service/eks/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Clusters() *schema.Table { diff --git a/resources/services/eks/clusters_mock_test.go b/resources/services/eks/clusters_mock_test.go index bd628074..a257d18e 100644 --- a/resources/services/eks/clusters_mock_test.go +++ b/resources/services/eks/clusters_mock_test.go @@ -3,14 +3,15 @@ package eks import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/eks" "github.com/aws/aws-sdk-go-v2/service/eks/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEksClusters(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/eks/fargate_profiles.go b/resources/services/eks/fargate_profiles.go index cafba1c3..e7bff567 100644 --- a/resources/services/eks/fargate_profiles.go +++ b/resources/services/eks/fargate_profiles.go @@ -3,12 +3,13 @@ package eks import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/eks" "github.com/aws/aws-sdk-go-v2/service/eks/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func fargateProfiles() *schema.Table { diff --git a/resources/services/eks/identity_provider_configs.go b/resources/services/eks/identity_provider_configs.go index 3d4d20e0..df12b2a3 100644 --- a/resources/services/eks/identity_provider_configs.go +++ b/resources/services/eks/identity_provider_configs.go @@ -3,13 +3,14 @@ package eks import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/eks" "github.com/aws/aws-sdk-go-v2/service/eks/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func identityProviderConfigs() *schema.Table { diff --git a/resources/services/eks/node_groups.go b/resources/services/eks/node_groups.go index d8a80f36..c0e1d2e6 100644 --- a/resources/services/eks/node_groups.go +++ b/resources/services/eks/node_groups.go @@ -3,12 +3,13 @@ package eks import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/eks" "github.com/aws/aws-sdk-go-v2/service/eks/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func nodeGroups() *schema.Table { diff --git a/resources/services/elasticache/clusters.go b/resources/services/elasticache/clusters.go index 7ba82551..88c4e030 100644 --- a/resources/services/elasticache/clusters.go +++ b/resources/services/elasticache/clusters.go @@ -3,14 +3,15 @@ package elasticache import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/elasticache" "github.com/aws/aws-sdk-go-v2/service/elasticache/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Clusters() *schema.Table { diff --git a/resources/services/elasticache/clusters_mock_test.go b/resources/services/elasticache/clusters_mock_test.go index 074dd380..1e92a614 100644 --- a/resources/services/elasticache/clusters_mock_test.go +++ b/resources/services/elasticache/clusters_mock_test.go @@ -3,13 +3,14 @@ package elasticache import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/elasticache" "github.com/aws/aws-sdk-go-v2/service/elasticache/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildElasticacheClusters(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/elasticache/global_replication_groups.go b/resources/services/elasticache/global_replication_groups.go index 85049700..2cc2f0db 100644 --- a/resources/services/elasticache/global_replication_groups.go +++ b/resources/services/elasticache/global_replication_groups.go @@ -3,12 +3,13 @@ package elasticache import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/elasticache" "github.com/aws/aws-sdk-go-v2/service/elasticache/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func GlobalReplicationGroups() *schema.Table { diff --git a/resources/services/elasticache/global_replication_groups_mock_test.go b/resources/services/elasticache/global_replication_groups_mock_test.go index 3b2e75c3..2a764047 100644 --- a/resources/services/elasticache/global_replication_groups_mock_test.go +++ b/resources/services/elasticache/global_replication_groups_mock_test.go @@ -3,12 +3,13 @@ package elasticache import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/elasticache" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildElasticacheGlobalReplicationGroup(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/elasticache/parameter_groups.go b/resources/services/elasticache/parameter_groups.go index 85992adb..a0661bce 100644 --- a/resources/services/elasticache/parameter_groups.go +++ b/resources/services/elasticache/parameter_groups.go @@ -3,12 +3,13 @@ package elasticache import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/elasticache" "github.com/aws/aws-sdk-go-v2/service/elasticache/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ParameterGroups() *schema.Table { diff --git a/resources/services/elasticache/parameter_groups_mock_test.go b/resources/services/elasticache/parameter_groups_mock_test.go index 04214a42..d045aeee 100644 --- a/resources/services/elasticache/parameter_groups_mock_test.go +++ b/resources/services/elasticache/parameter_groups_mock_test.go @@ -3,12 +3,13 @@ package elasticache import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/elasticache" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildElasticacheParameterGroups(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/elasticache/replication_groups.go b/resources/services/elasticache/replication_groups.go index e6ba5be4..1af950cb 100644 --- a/resources/services/elasticache/replication_groups.go +++ b/resources/services/elasticache/replication_groups.go @@ -3,13 +3,14 @@ package elasticache import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/elasticache" "github.com/aws/aws-sdk-go-v2/service/elasticache/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func ReplicationGroups() *schema.Table { diff --git a/resources/services/elasticache/replication_groups_mock_test.go b/resources/services/elasticache/replication_groups_mock_test.go index 74dfaea7..946dcf61 100644 --- a/resources/services/elasticache/replication_groups_mock_test.go +++ b/resources/services/elasticache/replication_groups_mock_test.go @@ -3,13 +3,14 @@ package elasticache import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/elasticache" "github.com/aws/aws-sdk-go-v2/service/elasticache/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildElasticacheReplicationGroups(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/elasticache/reserved_cache_nodes.go b/resources/services/elasticache/reserved_cache_nodes.go index b2981698..57522e8a 100644 --- a/resources/services/elasticache/reserved_cache_nodes.go +++ b/resources/services/elasticache/reserved_cache_nodes.go @@ -3,12 +3,13 @@ package elasticache import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/elasticache" "github.com/aws/aws-sdk-go-v2/service/elasticache/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ReservedCacheNodes() *schema.Table { diff --git a/resources/services/elasticache/reserved_cache_nodes_mock_test.go b/resources/services/elasticache/reserved_cache_nodes_mock_test.go index 4dd21fd2..ca08f7cb 100644 --- a/resources/services/elasticache/reserved_cache_nodes_mock_test.go +++ b/resources/services/elasticache/reserved_cache_nodes_mock_test.go @@ -3,12 +3,13 @@ package elasticache import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/elasticache" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildElasticacheReservedCacheNodes(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/elasticache/service_updates.go b/resources/services/elasticache/service_updates.go index 830274f4..554df36e 100644 --- a/resources/services/elasticache/service_updates.go +++ b/resources/services/elasticache/service_updates.go @@ -3,14 +3,15 @@ package elasticache import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/elasticache" "github.com/aws/aws-sdk-go-v2/service/elasticache/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ServiceUpdates() *schema.Table { diff --git a/resources/services/elasticache/service_updates_mock_test.go b/resources/services/elasticache/service_updates_mock_test.go index fe8f4f97..c097b003 100644 --- a/resources/services/elasticache/service_updates_mock_test.go +++ b/resources/services/elasticache/service_updates_mock_test.go @@ -3,12 +3,13 @@ package elasticache import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/elasticache" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildElasticacheServiceUpdates(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/elasticache/snapshots.go b/resources/services/elasticache/snapshots.go index 3f4d6db7..eeeb3ca1 100644 --- a/resources/services/elasticache/snapshots.go +++ b/resources/services/elasticache/snapshots.go @@ -3,12 +3,13 @@ package elasticache import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/elasticache" "github.com/aws/aws-sdk-go-v2/service/elasticache/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Snapshots() *schema.Table { diff --git a/resources/services/elasticache/snapshots_mock_test.go b/resources/services/elasticache/snapshots_mock_test.go index 642c820c..a3da4dae 100644 --- a/resources/services/elasticache/snapshots_mock_test.go +++ b/resources/services/elasticache/snapshots_mock_test.go @@ -3,12 +3,13 @@ package elasticache import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/elasticache" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildElasticacheSnapshots(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/elasticache/subnet_groups.go b/resources/services/elasticache/subnet_groups.go index ef964b0f..34166efb 100644 --- a/resources/services/elasticache/subnet_groups.go +++ b/resources/services/elasticache/subnet_groups.go @@ -3,12 +3,13 @@ package elasticache import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/elasticache" "github.com/aws/aws-sdk-go-v2/service/elasticache/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func SubnetGroups() *schema.Table { diff --git a/resources/services/elasticache/subnet_groups_mock_test.go b/resources/services/elasticache/subnet_groups_mock_test.go index 662f3697..bab02981 100644 --- a/resources/services/elasticache/subnet_groups_mock_test.go +++ b/resources/services/elasticache/subnet_groups_mock_test.go @@ -3,12 +3,13 @@ package elasticache import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/elasticache" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildElasticacheSubnetGroups(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/elasticache/user_groups.go b/resources/services/elasticache/user_groups.go index a5d90222..1548b1df 100644 --- a/resources/services/elasticache/user_groups.go +++ b/resources/services/elasticache/user_groups.go @@ -3,12 +3,13 @@ package elasticache import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/elasticache" "github.com/aws/aws-sdk-go-v2/service/elasticache/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func UserGroups() *schema.Table { diff --git a/resources/services/elasticache/user_groups_mock_test.go b/resources/services/elasticache/user_groups_mock_test.go index 71f7314d..d68476c0 100644 --- a/resources/services/elasticache/user_groups_mock_test.go +++ b/resources/services/elasticache/user_groups_mock_test.go @@ -3,12 +3,13 @@ package elasticache import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/elasticache" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildElasticacheUserGroups(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/elasticache/users.go b/resources/services/elasticache/users.go index ccdb8bf9..fd0fee83 100644 --- a/resources/services/elasticache/users.go +++ b/resources/services/elasticache/users.go @@ -3,12 +3,13 @@ package elasticache import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/elasticache" "github.com/aws/aws-sdk-go-v2/service/elasticache/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Users() *schema.Table { diff --git a/resources/services/elasticache/users_mock_test.go b/resources/services/elasticache/users_mock_test.go index 8d984b71..62561e93 100644 --- a/resources/services/elasticache/users_mock_test.go +++ b/resources/services/elasticache/users_mock_test.go @@ -3,12 +3,13 @@ package elasticache import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/elasticache" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildElasticacheUsers(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/elasticbeanstalk/application_versions.go b/resources/services/elasticbeanstalk/application_versions.go index f32f8ecc..3ca84c92 100644 --- a/resources/services/elasticbeanstalk/application_versions.go +++ b/resources/services/elasticbeanstalk/application_versions.go @@ -3,13 +3,14 @@ package elasticbeanstalk import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/elasticbeanstalk" "github.com/aws/aws-sdk-go-v2/service/elasticbeanstalk/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ApplicationVersions() *schema.Table { diff --git a/resources/services/elasticbeanstalk/application_versions_mock_test.go b/resources/services/elasticbeanstalk/application_versions_mock_test.go index 105fcf16..772a2673 100644 --- a/resources/services/elasticbeanstalk/application_versions_mock_test.go +++ b/resources/services/elasticbeanstalk/application_versions_mock_test.go @@ -3,13 +3,14 @@ package elasticbeanstalk import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/elasticbeanstalk" elasticbeanstalkTypes "github.com/aws/aws-sdk-go-v2/service/elasticbeanstalk/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildElasticbeanstalkApplicationVersions(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/elasticbeanstalk/applications.go b/resources/services/elasticbeanstalk/applications.go index 0b64ab78..5f3778c6 100644 --- a/resources/services/elasticbeanstalk/applications.go +++ b/resources/services/elasticbeanstalk/applications.go @@ -3,13 +3,14 @@ package elasticbeanstalk import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/elasticbeanstalk" "github.com/aws/aws-sdk-go-v2/service/elasticbeanstalk/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Applications() *schema.Table { diff --git a/resources/services/elasticbeanstalk/applications_mock_test.go b/resources/services/elasticbeanstalk/applications_mock_test.go index 1da6d054..6f2779e6 100644 --- a/resources/services/elasticbeanstalk/applications_mock_test.go +++ b/resources/services/elasticbeanstalk/applications_mock_test.go @@ -3,13 +3,14 @@ package elasticbeanstalk import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/elasticbeanstalk" elasticbeanstalkTypes "github.com/aws/aws-sdk-go-v2/service/elasticbeanstalk/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildElasticbeanstalkApplications(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/elasticbeanstalk/configuration_options.go b/resources/services/elasticbeanstalk/configuration_options.go index 529fe46b..52e6ecfc 100644 --- a/resources/services/elasticbeanstalk/configuration_options.go +++ b/resources/services/elasticbeanstalk/configuration_options.go @@ -4,15 +4,16 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/elasticbeanstalk/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/elasticbeanstalk" "github.com/aws/aws-sdk-go-v2/service/elasticbeanstalk/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/elasticbeanstalk/models" ) func configurationOptions() *schema.Table { diff --git a/resources/services/elasticbeanstalk/configuration_settings.go b/resources/services/elasticbeanstalk/configuration_settings.go index b1377622..3db9acb1 100644 --- a/resources/services/elasticbeanstalk/configuration_settings.go +++ b/resources/services/elasticbeanstalk/configuration_settings.go @@ -4,15 +4,16 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/elasticbeanstalk/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/elasticbeanstalk" "github.com/aws/aws-sdk-go-v2/service/elasticbeanstalk/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/elasticbeanstalk/models" ) func configurationSettings() *schema.Table { diff --git a/resources/services/elasticbeanstalk/environments.go b/resources/services/elasticbeanstalk/environments.go index 7e06f45c..c564805f 100644 --- a/resources/services/elasticbeanstalk/environments.go +++ b/resources/services/elasticbeanstalk/environments.go @@ -3,14 +3,15 @@ package elasticbeanstalk import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/elasticbeanstalk" "github.com/aws/aws-sdk-go-v2/service/elasticbeanstalk/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Environments() *schema.Table { @@ -75,6 +76,7 @@ func fetchElasticbeanstalkEnvironments(ctx context.Context, meta schema.ClientMe } return nil } + func resolveElasticbeanstalkEnvironmentListeners(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { p := resource.Item.(types.EnvironmentDescription) if p.Resources == nil || p.Resources.LoadBalancer == nil { @@ -86,6 +88,7 @@ func resolveElasticbeanstalkEnvironmentListeners(ctx context.Context, meta schem } return resource.Set(c.Name, listeners) } + func resolveElasticbeanstalkEnvironmentTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { p := resource.Item.(types.EnvironmentDescription) cl := meta.(*client.Client) diff --git a/resources/services/elasticbeanstalk/envrionments_mock_test.go b/resources/services/elasticbeanstalk/envrionments_mock_test.go index 1d36564c..bed49731 100644 --- a/resources/services/elasticbeanstalk/envrionments_mock_test.go +++ b/resources/services/elasticbeanstalk/envrionments_mock_test.go @@ -3,13 +3,14 @@ package elasticbeanstalk import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/elasticbeanstalk" elasticbeanstalkTypes "github.com/aws/aws-sdk-go-v2/service/elasticbeanstalk/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildElasticbeanstalkEnvironments(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/elasticsearch/domains.go b/resources/services/elasticsearch/domains.go index 1a0b7304..08e0a5be 100644 --- a/resources/services/elasticsearch/domains.go +++ b/resources/services/elasticsearch/domains.go @@ -3,14 +3,15 @@ package elasticsearch import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/elasticsearchservice" "github.com/aws/aws-sdk-go-v2/service/elasticsearchservice/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Domains() *schema.Table { diff --git a/resources/services/elasticsearch/domains_mock_test.go b/resources/services/elasticsearch/domains_mock_test.go index 71a63ee6..b7cb1a25 100644 --- a/resources/services/elasticsearch/domains_mock_test.go +++ b/resources/services/elasticsearch/domains_mock_test.go @@ -3,13 +3,14 @@ package elasticsearch import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/elasticsearchservice" "github.com/aws/aws-sdk-go-v2/service/elasticsearchservice/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildElasticSearchDomains(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/elastictranscoder/pipeline_jobs.go b/resources/services/elastictranscoder/pipeline_jobs.go index bea2f168..444caa5d 100644 --- a/resources/services/elastictranscoder/pipeline_jobs.go +++ b/resources/services/elastictranscoder/pipeline_jobs.go @@ -3,13 +3,14 @@ package elastictranscoder import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/elastictranscoder" "github.com/aws/aws-sdk-go-v2/service/elastictranscoder/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func pipelineJobs() *schema.Table { diff --git a/resources/services/elastictranscoder/pipelines.go b/resources/services/elastictranscoder/pipelines.go index f481302a..64e8ac45 100644 --- a/resources/services/elastictranscoder/pipelines.go +++ b/resources/services/elastictranscoder/pipelines.go @@ -3,12 +3,13 @@ package elastictranscoder import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/elastictranscoder" "github.com/aws/aws-sdk-go-v2/service/elastictranscoder/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Pipelines() *schema.Table { diff --git a/resources/services/elastictranscoder/pipelines_mock_test.go b/resources/services/elastictranscoder/pipelines_mock_test.go index e5611507..e1396635 100644 --- a/resources/services/elastictranscoder/pipelines_mock_test.go +++ b/resources/services/elastictranscoder/pipelines_mock_test.go @@ -3,13 +3,14 @@ package elastictranscoder import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/elastictranscoder" "github.com/aws/aws-sdk-go-v2/service/elastictranscoder/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildElastictranscoderPipelinesMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -37,6 +38,7 @@ func buildElastictranscoderPipelinesMock(t *testing.T, ctrl *gomock.Controller) Elastictranscoder: m, } } + func TestElastictranscoderPipelines(t *testing.T) { client.AwsMockTestHelper(t, Pipelines(), buildElastictranscoderPipelinesMock, client.TestOptions{}) } diff --git a/resources/services/elastictranscoder/presets.go b/resources/services/elastictranscoder/presets.go index 7bad67b7..f4bdbd7b 100644 --- a/resources/services/elastictranscoder/presets.go +++ b/resources/services/elastictranscoder/presets.go @@ -3,12 +3,13 @@ package elastictranscoder import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/elastictranscoder" "github.com/aws/aws-sdk-go-v2/service/elastictranscoder/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Presets() *schema.Table { diff --git a/resources/services/elastictranscoder/presets_mock_test.go b/resources/services/elastictranscoder/presets_mock_test.go index b824754a..7784bb0d 100644 --- a/resources/services/elastictranscoder/presets_mock_test.go +++ b/resources/services/elastictranscoder/presets_mock_test.go @@ -3,13 +3,14 @@ package elastictranscoder import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/elastictranscoder" "github.com/aws/aws-sdk-go-v2/service/elastictranscoder/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildElastictranscoderPresetsMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -26,6 +27,7 @@ func buildElastictranscoderPresetsMock(t *testing.T, ctrl *gomock.Controller) cl Elastictranscoder: m, } } + func TestElastictranscoderPresets(t *testing.T) { client.AwsMockTestHelper(t, Presets(), buildElastictranscoderPresetsMock, client.TestOptions{}) } diff --git a/resources/services/elbv1/load_balancer_policies.go b/resources/services/elbv1/load_balancer_policies.go index 7e4c526c..ae63bb34 100644 --- a/resources/services/elbv1/load_balancer_policies.go +++ b/resources/services/elbv1/load_balancer_policies.go @@ -3,14 +3,15 @@ package elbv1 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/elbv1/models" "github.com/apache/arrow/go/v16/arrow" elbv1 "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing" "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/elbv1/models" ) func loadBalancerPolicies() *schema.Table { @@ -56,6 +57,7 @@ func fetchElbv1LoadBalancerPolicies(ctx context.Context, meta schema.ClientMeta, res <- response.PolicyDescriptions return nil } + func resolveElbv1loadBalancerPolicyAttributeDescriptions(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { r := resource.Item.(types.PolicyDescription) diff --git a/resources/services/elbv1/load_balancers.go b/resources/services/elbv1/load_balancers.go index 465098b1..2c8a1b63 100644 --- a/resources/services/elbv1/load_balancers.go +++ b/resources/services/elbv1/load_balancers.go @@ -3,13 +3,14 @@ package elbv1 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/elbv1/models" "github.com/apache/arrow/go/v16/arrow" elbv1 "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing" "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/elbv1/models" ) func LoadBalancers() *schema.Table { diff --git a/resources/services/elbv1/load_balancers_mock_test.go b/resources/services/elbv1/load_balancers_mock_test.go index f4d78a14..c52c22a3 100644 --- a/resources/services/elbv1/load_balancers_mock_test.go +++ b/resources/services/elbv1/load_balancers_mock_test.go @@ -3,13 +3,14 @@ package elbv1 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing" elbv1Types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildElbv1LoadBalancers(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/elbv2/listener_certificates.go b/resources/services/elbv2/listener_certificates.go index 7682cb23..f48bd175 100644 --- a/resources/services/elbv2/listener_certificates.go +++ b/resources/services/elbv2/listener_certificates.go @@ -3,13 +3,14 @@ package elbv2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" elbv2 "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func listenerCertificates() *schema.Table { diff --git a/resources/services/elbv2/listener_rules.go b/resources/services/elbv2/listener_rules.go index 0372c085..16cc38c5 100644 --- a/resources/services/elbv2/listener_rules.go +++ b/resources/services/elbv2/listener_rules.go @@ -3,13 +3,14 @@ package elbv2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" elbv2 "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func listenerRules() *schema.Table { diff --git a/resources/services/elbv2/listeners.go b/resources/services/elbv2/listeners.go index b16423f7..00bc4607 100644 --- a/resources/services/elbv2/listeners.go +++ b/resources/services/elbv2/listeners.go @@ -3,13 +3,14 @@ package elbv2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" elbv2 "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func listeners() *schema.Table { diff --git a/resources/services/elbv2/load_balancer_attributes.go b/resources/services/elbv2/load_balancer_attributes.go index 17d92fd1..ce06dd7e 100644 --- a/resources/services/elbv2/load_balancer_attributes.go +++ b/resources/services/elbv2/load_balancer_attributes.go @@ -3,12 +3,13 @@ package elbv2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" elbv2 "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func loadBalancerAttributes() *schema.Table { diff --git a/resources/services/elbv2/load_balancer_web_acls.go b/resources/services/elbv2/load_balancer_web_acls.go index 8b130910..81624454 100644 --- a/resources/services/elbv2/load_balancer_web_acls.go +++ b/resources/services/elbv2/load_balancer_web_acls.go @@ -10,8 +10,8 @@ import ( wafv2types "github.com/aws/aws-sdk-go-v2/service/wafv2/types" "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func webACLs() *schema.Table { diff --git a/resources/services/elbv2/load_balancers.go b/resources/services/elbv2/load_balancers.go index 3f79e1de..cac900b2 100644 --- a/resources/services/elbv2/load_balancers.go +++ b/resources/services/elbv2/load_balancers.go @@ -3,13 +3,14 @@ package elbv2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" elbv2 "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func LoadBalancers() *schema.Table { diff --git a/resources/services/elbv2/load_balancers_mock_test.go b/resources/services/elbv2/load_balancers_mock_test.go index 40930890..e7b0aaa9 100644 --- a/resources/services/elbv2/load_balancers_mock_test.go +++ b/resources/services/elbv2/load_balancers_mock_test.go @@ -3,15 +3,16 @@ package elbv2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" elbv2Types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" "github.com/aws/aws-sdk-go-v2/service/wafv2" "github.com/aws/aws-sdk-go-v2/service/wafv2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildLoadBalancers(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/elbv2/target_group_target_health_descriptions.go b/resources/services/elbv2/target_group_target_health_descriptions.go index 511c6e23..bac8f14a 100644 --- a/resources/services/elbv2/target_group_target_health_descriptions.go +++ b/resources/services/elbv2/target_group_target_health_descriptions.go @@ -3,12 +3,13 @@ package elbv2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" elbv2 "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func targetGroupTargetHealthDescriptions() *schema.Table { diff --git a/resources/services/elbv2/target_groups.go b/resources/services/elbv2/target_groups.go index ac305af6..d2141a89 100644 --- a/resources/services/elbv2/target_groups.go +++ b/resources/services/elbv2/target_groups.go @@ -3,13 +3,14 @@ package elbv2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" elbv2 "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func TargetGroups() *schema.Table { diff --git a/resources/services/elbv2/target_groups_mock_test.go b/resources/services/elbv2/target_groups_mock_test.go index 58785e1b..cba39eeb 100644 --- a/resources/services/elbv2/target_groups_mock_test.go +++ b/resources/services/elbv2/target_groups_mock_test.go @@ -3,13 +3,14 @@ package elbv2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" elbv2Types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildTargetGroups(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/emr/cluster_instance_fleets.go b/resources/services/emr/cluster_instance_fleets.go index 5fbdbd4e..ce0d90d4 100644 --- a/resources/services/emr/cluster_instance_fleets.go +++ b/resources/services/emr/cluster_instance_fleets.go @@ -3,12 +3,13 @@ package emr import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/emr" "github.com/aws/aws-sdk-go-v2/service/emr/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func clusterInstanceFleets() *schema.Table { diff --git a/resources/services/emr/cluster_instance_groups.go b/resources/services/emr/cluster_instance_groups.go index 5521b26f..3c02c35d 100644 --- a/resources/services/emr/cluster_instance_groups.go +++ b/resources/services/emr/cluster_instance_groups.go @@ -3,12 +3,13 @@ package emr import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/emr" "github.com/aws/aws-sdk-go-v2/service/emr/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func clusterInstanceGroups() *schema.Table { diff --git a/resources/services/emr/cluster_instances.go b/resources/services/emr/cluster_instances.go index 903f4d72..77d9aabd 100644 --- a/resources/services/emr/cluster_instances.go +++ b/resources/services/emr/cluster_instances.go @@ -3,14 +3,15 @@ package emr import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/emr" "github.com/aws/aws-sdk-go-v2/service/emr/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func clusterInstances() *schema.Table { diff --git a/resources/services/emr/clusters.go b/resources/services/emr/clusters.go index aa405115..6b06a8c4 100644 --- a/resources/services/emr/clusters.go +++ b/resources/services/emr/clusters.go @@ -3,13 +3,14 @@ package emr import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/emr" "github.com/aws/aws-sdk-go-v2/service/emr/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Clusters() *schema.Table { diff --git a/resources/services/emr/clusters_mock_test.go b/resources/services/emr/clusters_mock_test.go index ecc5afc0..d4e67be1 100644 --- a/resources/services/emr/clusters_mock_test.go +++ b/resources/services/emr/clusters_mock_test.go @@ -3,14 +3,15 @@ package emr import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/emr" "github.com/aws/aws-sdk-go-v2/service/emr/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEMRClusters(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/emr/notebook_executions.go b/resources/services/emr/notebook_executions.go index bb47c995..33200e7e 100644 --- a/resources/services/emr/notebook_executions.go +++ b/resources/services/emr/notebook_executions.go @@ -3,13 +3,14 @@ package emr import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/emr" "github.com/aws/aws-sdk-go-v2/service/emr/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func notebookExecutions() *schema.Table { diff --git a/resources/services/emr/steps.go b/resources/services/emr/steps.go index b2da233b..d0d6c4f2 100644 --- a/resources/services/emr/steps.go +++ b/resources/services/emr/steps.go @@ -3,12 +3,13 @@ package emr import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/emr" "github.com/aws/aws-sdk-go-v2/service/emr/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func steps() *schema.Table { diff --git a/resources/services/emr/studio_session_mapping.go b/resources/services/emr/studio_session_mapping.go index c659a7ab..2bdf8876 100644 --- a/resources/services/emr/studio_session_mapping.go +++ b/resources/services/emr/studio_session_mapping.go @@ -3,12 +3,13 @@ package emr import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/emr" "github.com/aws/aws-sdk-go-v2/service/emr/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func studioSessionMapping() *schema.Table { diff --git a/resources/services/emr/studios.go b/resources/services/emr/studios.go index 3c95a6c1..7e2787de 100644 --- a/resources/services/emr/studios.go +++ b/resources/services/emr/studios.go @@ -3,13 +3,14 @@ package emr import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/emr" "github.com/aws/aws-sdk-go-v2/service/emr/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Studios() *schema.Table { diff --git a/resources/services/emr/studios_test.go b/resources/services/emr/studios_test.go index 60b5566f..e7a0b9e6 100644 --- a/resources/services/emr/studios_test.go +++ b/resources/services/emr/studios_test.go @@ -1,14 +1,16 @@ package emr import ( - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "testing" + "github.com/aws/aws-sdk-go-v2/service/emr" "github.com/aws/aws-sdk-go-v2/service/emr/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" - "testing" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildStudios(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/eventbridge/api_destinations.go b/resources/services/eventbridge/api_destinations.go index b684f21a..ce06e8a7 100644 --- a/resources/services/eventbridge/api_destinations.go +++ b/resources/services/eventbridge/api_destinations.go @@ -3,13 +3,14 @@ package eventbridge import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/eventbridge" "github.com/aws/aws-sdk-go-v2/service/eventbridge/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ApiDestinations() *schema.Table { diff --git a/resources/services/eventbridge/api_destinations_mock_test.go b/resources/services/eventbridge/api_destinations_mock_test.go index 9089c731..450f060a 100644 --- a/resources/services/eventbridge/api_destinations_mock_test.go +++ b/resources/services/eventbridge/api_destinations_mock_test.go @@ -3,13 +3,14 @@ package eventbridge import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/eventbridge" "github.com/aws/aws-sdk-go-v2/service/eventbridge/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEventbridgeApiDestinationsMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -29,6 +30,7 @@ func buildEventbridgeApiDestinationsMock(t *testing.T, ctrl *gomock.Controller) Eventbridge: m, } } + func TestEventbridgeApiDestinations(t *testing.T) { client.AwsMockTestHelper(t, ApiDestinations(), buildEventbridgeApiDestinationsMock, client.TestOptions{}) } diff --git a/resources/services/eventbridge/archives.go b/resources/services/eventbridge/archives.go index 254e47d2..5b5350d5 100644 --- a/resources/services/eventbridge/archives.go +++ b/resources/services/eventbridge/archives.go @@ -3,14 +3,15 @@ package eventbridge import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/eventbridge" "github.com/aws/aws-sdk-go-v2/service/eventbridge/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Archives() *schema.Table { diff --git a/resources/services/eventbridge/archives_mock_test.go b/resources/services/eventbridge/archives_mock_test.go index cccbd6e3..8a43267b 100644 --- a/resources/services/eventbridge/archives_mock_test.go +++ b/resources/services/eventbridge/archives_mock_test.go @@ -3,13 +3,14 @@ package eventbridge import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/eventbridge" "github.com/aws/aws-sdk-go-v2/service/eventbridge/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEventbridgeArchivesMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -29,6 +30,7 @@ func buildEventbridgeArchivesMock(t *testing.T, ctrl *gomock.Controller) client. Eventbridge: m, } } + func TestEventbridgeArchives(t *testing.T) { client.AwsMockTestHelper(t, Archives(), buildEventbridgeArchivesMock, client.TestOptions{}) } diff --git a/resources/services/eventbridge/connections.go b/resources/services/eventbridge/connections.go index 6a4c47cf..05565e07 100644 --- a/resources/services/eventbridge/connections.go +++ b/resources/services/eventbridge/connections.go @@ -3,13 +3,14 @@ package eventbridge import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/eventbridge" "github.com/aws/aws-sdk-go-v2/service/eventbridge/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Connections() *schema.Table { diff --git a/resources/services/eventbridge/connections_mock_test.go b/resources/services/eventbridge/connections_mock_test.go index 2accd2ac..86bee214 100644 --- a/resources/services/eventbridge/connections_mock_test.go +++ b/resources/services/eventbridge/connections_mock_test.go @@ -3,13 +3,14 @@ package eventbridge import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/eventbridge" "github.com/aws/aws-sdk-go-v2/service/eventbridge/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEventbridgeConnectionsMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -29,6 +30,7 @@ func buildEventbridgeConnectionsMock(t *testing.T, ctrl *gomock.Controller) clie Eventbridge: m, } } + func TestEventbridgeConnections(t *testing.T) { client.AwsMockTestHelper(t, Connections(), buildEventbridgeConnectionsMock, client.TestOptions{}) } diff --git a/resources/services/eventbridge/endpoints.go b/resources/services/eventbridge/endpoints.go index 3d322c70..f85e200f 100644 --- a/resources/services/eventbridge/endpoints.go +++ b/resources/services/eventbridge/endpoints.go @@ -3,13 +3,14 @@ package eventbridge import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/eventbridge" "github.com/aws/aws-sdk-go-v2/service/eventbridge/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Endpoints() *schema.Table { diff --git a/resources/services/eventbridge/endpoints_mock_test.go b/resources/services/eventbridge/endpoints_mock_test.go index 057a0236..378aa671 100644 --- a/resources/services/eventbridge/endpoints_mock_test.go +++ b/resources/services/eventbridge/endpoints_mock_test.go @@ -3,13 +3,14 @@ package eventbridge import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/eventbridge" "github.com/aws/aws-sdk-go-v2/service/eventbridge/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEventbridgeEndpointsMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -29,6 +30,7 @@ func buildEventbridgeEndpointsMock(t *testing.T, ctrl *gomock.Controller) client Eventbridge: m, } } + func TestEventbridgeEndpoints(t *testing.T) { client.AwsMockTestHelper(t, Endpoints(), buildEventbridgeEndpointsMock, client.TestOptions{}) } diff --git a/resources/services/eventbridge/event_bus_rules.go b/resources/services/eventbridge/event_bus_rules.go index c228e22d..76b29b75 100644 --- a/resources/services/eventbridge/event_bus_rules.go +++ b/resources/services/eventbridge/event_bus_rules.go @@ -3,14 +3,15 @@ package eventbridge import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/eventbridge" "github.com/aws/aws-sdk-go-v2/service/eventbridge/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func eventBusRules() *schema.Table { diff --git a/resources/services/eventbridge/event_bus_targets.go b/resources/services/eventbridge/event_bus_targets.go index a90f3999..3ccb1fdc 100644 --- a/resources/services/eventbridge/event_bus_targets.go +++ b/resources/services/eventbridge/event_bus_targets.go @@ -3,13 +3,14 @@ package eventbridge import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/eventbridge" "github.com/aws/aws-sdk-go-v2/service/eventbridge/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func eventBusTargets() *schema.Table { diff --git a/resources/services/eventbridge/event_buses.go b/resources/services/eventbridge/event_buses.go index 1234e37d..49229b89 100644 --- a/resources/services/eventbridge/event_buses.go +++ b/resources/services/eventbridge/event_buses.go @@ -3,14 +3,15 @@ package eventbridge import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/eventbridge" "github.com/aws/aws-sdk-go-v2/service/eventbridge/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func EventBuses() *schema.Table { diff --git a/resources/services/eventbridge/event_buses_mock_test.go b/resources/services/eventbridge/event_buses_mock_test.go index a1d0f2a1..b67206ee 100644 --- a/resources/services/eventbridge/event_buses_mock_test.go +++ b/resources/services/eventbridge/event_buses_mock_test.go @@ -3,13 +3,14 @@ package eventbridge import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/eventbridge" "github.com/aws/aws-sdk-go-v2/service/eventbridge/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEventBridgeEventBusesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/eventbridge/event_sources.go b/resources/services/eventbridge/event_sources.go index fd2dff69..c65b0a05 100644 --- a/resources/services/eventbridge/event_sources.go +++ b/resources/services/eventbridge/event_sources.go @@ -3,13 +3,14 @@ package eventbridge import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/eventbridge" "github.com/aws/aws-sdk-go-v2/service/eventbridge/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func EventSources() *schema.Table { diff --git a/resources/services/eventbridge/event_sources_mock_test.go b/resources/services/eventbridge/event_sources_mock_test.go index d6c4337d..7ab1b455 100644 --- a/resources/services/eventbridge/event_sources_mock_test.go +++ b/resources/services/eventbridge/event_sources_mock_test.go @@ -3,13 +3,14 @@ package eventbridge import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/eventbridge" "github.com/aws/aws-sdk-go-v2/service/eventbridge/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEventbridgeEventSourcesMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -29,6 +30,7 @@ func buildEventbridgeEventSourcesMock(t *testing.T, ctrl *gomock.Controller) cli Eventbridge: m, } } + func TestEventbridgeEventSources(t *testing.T) { client.AwsMockTestHelper(t, EventSources(), buildEventbridgeEventSourcesMock, client.TestOptions{}) } diff --git a/resources/services/eventbridge/replays.go b/resources/services/eventbridge/replays.go index feb318aa..da388326 100644 --- a/resources/services/eventbridge/replays.go +++ b/resources/services/eventbridge/replays.go @@ -3,13 +3,14 @@ package eventbridge import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/eventbridge" "github.com/aws/aws-sdk-go-v2/service/eventbridge/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Replays() *schema.Table { diff --git a/resources/services/eventbridge/replays_mock_test.go b/resources/services/eventbridge/replays_mock_test.go index ebb3d905..a0d28484 100644 --- a/resources/services/eventbridge/replays_mock_test.go +++ b/resources/services/eventbridge/replays_mock_test.go @@ -3,13 +3,14 @@ package eventbridge import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/eventbridge" "github.com/aws/aws-sdk-go-v2/service/eventbridge/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEventbridgeReplaysMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -36,6 +37,7 @@ func buildEventbridgeReplaysMock(t *testing.T, ctrl *gomock.Controller) client.S Eventbridge: m, } } + func TestEventbridgeReplays(t *testing.T) { client.AwsMockTestHelper(t, Replays(), buildEventbridgeReplaysMock, client.TestOptions{}) } diff --git a/resources/services/firehose/delivery_streams.go b/resources/services/firehose/delivery_streams.go index 1020a3e1..ce05e3d4 100644 --- a/resources/services/firehose/delivery_streams.go +++ b/resources/services/firehose/delivery_streams.go @@ -3,14 +3,15 @@ package firehose import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/firehose" "github.com/aws/aws-sdk-go-v2/service/firehose/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func DeliveryStreams() *schema.Table { diff --git a/resources/services/firehose/delivery_streams_mock_test.go b/resources/services/firehose/delivery_streams_mock_test.go index 5432965f..22c924da 100644 --- a/resources/services/firehose/delivery_streams_mock_test.go +++ b/resources/services/firehose/delivery_streams_mock_test.go @@ -3,14 +3,15 @@ package firehose import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/firehose" "github.com/aws/aws-sdk-go-v2/service/firehose/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildKinesisFirehoses(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/frauddetector/batch_imports.go b/resources/services/frauddetector/batch_imports.go index 9fe1ccba..e1a44421 100644 --- a/resources/services/frauddetector/batch_imports.go +++ b/resources/services/frauddetector/batch_imports.go @@ -3,12 +3,13 @@ package frauddetector import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/frauddetector" "github.com/aws/aws-sdk-go-v2/service/frauddetector/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func BatchImports() *schema.Table { diff --git a/resources/services/frauddetector/batch_imports_mock_test.go b/resources/services/frauddetector/batch_imports_mock_test.go index 30e96287..b45821dd 100644 --- a/resources/services/frauddetector/batch_imports_mock_test.go +++ b/resources/services/frauddetector/batch_imports_mock_test.go @@ -3,13 +3,14 @@ package frauddetector import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/frauddetector" "github.com/aws/aws-sdk-go-v2/service/frauddetector/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildBatchImports(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/frauddetector/batch_predictions.go b/resources/services/frauddetector/batch_predictions.go index d55e9ebc..4be64090 100644 --- a/resources/services/frauddetector/batch_predictions.go +++ b/resources/services/frauddetector/batch_predictions.go @@ -3,12 +3,13 @@ package frauddetector import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/frauddetector" "github.com/aws/aws-sdk-go-v2/service/frauddetector/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func BatchPredictions() *schema.Table { diff --git a/resources/services/frauddetector/batch_predictions_mock_test.go b/resources/services/frauddetector/batch_predictions_mock_test.go index f00abb50..ed154e59 100644 --- a/resources/services/frauddetector/batch_predictions_mock_test.go +++ b/resources/services/frauddetector/batch_predictions_mock_test.go @@ -3,13 +3,14 @@ package frauddetector import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/frauddetector" "github.com/aws/aws-sdk-go-v2/service/frauddetector/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildBatchPredictions(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/frauddetector/detectors.go b/resources/services/frauddetector/detectors.go index 11ee01ef..5cb0e478 100644 --- a/resources/services/frauddetector/detectors.go +++ b/resources/services/frauddetector/detectors.go @@ -3,13 +3,14 @@ package frauddetector import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/frauddetector" "github.com/aws/aws-sdk-go-v2/service/frauddetector/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Detectors() *schema.Table { diff --git a/resources/services/frauddetector/detectors_mock_test.go b/resources/services/frauddetector/detectors_mock_test.go index 83440a26..ef9d0a72 100644 --- a/resources/services/frauddetector/detectors_mock_test.go +++ b/resources/services/frauddetector/detectors_mock_test.go @@ -3,13 +3,14 @@ package frauddetector import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/frauddetector" "github.com/aws/aws-sdk-go-v2/service/frauddetector/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDetectors(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/frauddetector/entity_types.go b/resources/services/frauddetector/entity_types.go index bd694207..c353fbd4 100644 --- a/resources/services/frauddetector/entity_types.go +++ b/resources/services/frauddetector/entity_types.go @@ -3,13 +3,14 @@ package frauddetector import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/frauddetector" "github.com/aws/aws-sdk-go-v2/service/frauddetector/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func EntityTypes() *schema.Table { diff --git a/resources/services/frauddetector/entity_types_mock_test.go b/resources/services/frauddetector/entity_types_mock_test.go index 13e8e676..6e42c01a 100644 --- a/resources/services/frauddetector/entity_types_mock_test.go +++ b/resources/services/frauddetector/entity_types_mock_test.go @@ -3,13 +3,14 @@ package frauddetector import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/frauddetector" "github.com/aws/aws-sdk-go-v2/service/frauddetector/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEntityTypes(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/frauddetector/event_types.go b/resources/services/frauddetector/event_types.go index 9fbb5beb..b5c437da 100644 --- a/resources/services/frauddetector/event_types.go +++ b/resources/services/frauddetector/event_types.go @@ -3,13 +3,14 @@ package frauddetector import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/frauddetector" "github.com/aws/aws-sdk-go-v2/service/frauddetector/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func EventTypes() *schema.Table { diff --git a/resources/services/frauddetector/event_types_mock_test.go b/resources/services/frauddetector/event_types_mock_test.go index f11813c5..98b58716 100644 --- a/resources/services/frauddetector/event_types_mock_test.go +++ b/resources/services/frauddetector/event_types_mock_test.go @@ -3,13 +3,14 @@ package frauddetector import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/frauddetector" "github.com/aws/aws-sdk-go-v2/service/frauddetector/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEventTypes(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/frauddetector/external_models.go b/resources/services/frauddetector/external_models.go index f191ef93..96c6dda3 100644 --- a/resources/services/frauddetector/external_models.go +++ b/resources/services/frauddetector/external_models.go @@ -3,12 +3,13 @@ package frauddetector import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/frauddetector" "github.com/aws/aws-sdk-go-v2/service/frauddetector/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ExternalModels() *schema.Table { diff --git a/resources/services/frauddetector/external_models_mock_test.go b/resources/services/frauddetector/external_models_mock_test.go index 97020f21..fb2d773c 100644 --- a/resources/services/frauddetector/external_models_mock_test.go +++ b/resources/services/frauddetector/external_models_mock_test.go @@ -3,13 +3,14 @@ package frauddetector import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/frauddetector" "github.com/aws/aws-sdk-go-v2/service/frauddetector/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildExternalModels(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/frauddetector/labels.go b/resources/services/frauddetector/labels.go index 1233ba69..ee884705 100644 --- a/resources/services/frauddetector/labels.go +++ b/resources/services/frauddetector/labels.go @@ -6,11 +6,11 @@ import ( "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/frauddetector" "github.com/aws/aws-sdk-go-v2/service/frauddetector/types" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Labels() *schema.Table { diff --git a/resources/services/frauddetector/labels_mock_test.go b/resources/services/frauddetector/labels_mock_test.go index ba2bc0ab..2a5b4496 100644 --- a/resources/services/frauddetector/labels_mock_test.go +++ b/resources/services/frauddetector/labels_mock_test.go @@ -3,13 +3,14 @@ package frauddetector import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/frauddetector" "github.com/aws/aws-sdk-go-v2/service/frauddetector/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildLabels(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/frauddetector/model_versions.go b/resources/services/frauddetector/model_versions.go index 7e9a065a..ea7876bc 100644 --- a/resources/services/frauddetector/model_versions.go +++ b/resources/services/frauddetector/model_versions.go @@ -3,12 +3,13 @@ package frauddetector import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/frauddetector" "github.com/aws/aws-sdk-go-v2/service/frauddetector/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func modelVersions() *schema.Table { diff --git a/resources/services/frauddetector/model_versions_mock_test.go b/resources/services/frauddetector/model_versions_mock_test.go index 33e80681..db2d1ec4 100644 --- a/resources/services/frauddetector/model_versions_mock_test.go +++ b/resources/services/frauddetector/model_versions_mock_test.go @@ -3,12 +3,13 @@ package frauddetector import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/frauddetector" "github.com/aws/aws-sdk-go-v2/service/frauddetector/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildModelVersions(t *testing.T, client *mocks.MockFrauddetectorClient) { diff --git a/resources/services/frauddetector/models.go b/resources/services/frauddetector/models.go index 5f380dea..f56c9f79 100644 --- a/resources/services/frauddetector/models.go +++ b/resources/services/frauddetector/models.go @@ -3,12 +3,13 @@ package frauddetector import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/frauddetector" "github.com/aws/aws-sdk-go-v2/service/frauddetector/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Models() *schema.Table { diff --git a/resources/services/frauddetector/models_mock_test.go b/resources/services/frauddetector/models_mock_test.go index ac2bc4c0..e7a414d7 100644 --- a/resources/services/frauddetector/models_mock_test.go +++ b/resources/services/frauddetector/models_mock_test.go @@ -3,13 +3,14 @@ package frauddetector import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/frauddetector" "github.com/aws/aws-sdk-go-v2/service/frauddetector/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildModels(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/frauddetector/outcomes.go b/resources/services/frauddetector/outcomes.go index 27489c6f..d1c26922 100644 --- a/resources/services/frauddetector/outcomes.go +++ b/resources/services/frauddetector/outcomes.go @@ -3,13 +3,14 @@ package frauddetector import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/frauddetector" "github.com/aws/aws-sdk-go-v2/service/frauddetector/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Outcomes() *schema.Table { diff --git a/resources/services/frauddetector/outcomes_mock_test.go b/resources/services/frauddetector/outcomes_mock_test.go index d35da80b..eff37646 100644 --- a/resources/services/frauddetector/outcomes_mock_test.go +++ b/resources/services/frauddetector/outcomes_mock_test.go @@ -3,13 +3,14 @@ package frauddetector import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/frauddetector" "github.com/aws/aws-sdk-go-v2/service/frauddetector/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildOutcomes(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/frauddetector/rules.go b/resources/services/frauddetector/rules.go index e4f1f94d..48f16a86 100644 --- a/resources/services/frauddetector/rules.go +++ b/resources/services/frauddetector/rules.go @@ -3,12 +3,13 @@ package frauddetector import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/frauddetector" "github.com/aws/aws-sdk-go-v2/service/frauddetector/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func rules() *schema.Table { diff --git a/resources/services/frauddetector/rules_mock_test.go b/resources/services/frauddetector/rules_mock_test.go index 08b64930..ca522367 100644 --- a/resources/services/frauddetector/rules_mock_test.go +++ b/resources/services/frauddetector/rules_mock_test.go @@ -3,12 +3,13 @@ package frauddetector import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/frauddetector" "github.com/aws/aws-sdk-go-v2/service/frauddetector/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRules(t *testing.T, client *mocks.MockFrauddetectorClient) { diff --git a/resources/services/frauddetector/tag_fetch.go b/resources/services/frauddetector/tag_fetch.go index 8fae3b2f..2e7e7b06 100644 --- a/resources/services/frauddetector/tag_fetch.go +++ b/resources/services/frauddetector/tag_fetch.go @@ -3,10 +3,11 @@ package frauddetector import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/frauddetector" - "github.com/cloudquery/plugin-sdk/v4/schema" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" ) func resolveResourceTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, column schema.Column) error { diff --git a/resources/services/frauddetector/tags_mock_test.go b/resources/services/frauddetector/tags_mock_test.go index 1657688c..c362145e 100644 --- a/resources/services/frauddetector/tags_mock_test.go +++ b/resources/services/frauddetector/tags_mock_test.go @@ -3,12 +3,13 @@ package frauddetector import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/frauddetector" "github.com/aws/aws-sdk-go-v2/service/frauddetector/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func addTagsCall(t *testing.T, client *mocks.MockFrauddetectorClient) { diff --git a/resources/services/frauddetector/variables.go b/resources/services/frauddetector/variables.go index 09d8b9b7..853c54f2 100644 --- a/resources/services/frauddetector/variables.go +++ b/resources/services/frauddetector/variables.go @@ -3,13 +3,14 @@ package frauddetector import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/frauddetector" "github.com/aws/aws-sdk-go-v2/service/frauddetector/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Variables() *schema.Table { diff --git a/resources/services/frauddetector/variables_mock_test.go b/resources/services/frauddetector/variables_mock_test.go index e883d03c..d53ce408 100644 --- a/resources/services/frauddetector/variables_mock_test.go +++ b/resources/services/frauddetector/variables_mock_test.go @@ -3,13 +3,14 @@ package frauddetector import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/frauddetector" "github.com/aws/aws-sdk-go-v2/service/frauddetector/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildVariables(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/fsx/data_repository_associations.go b/resources/services/fsx/data_repository_associations.go index 3b2f3639..5f55df4e 100644 --- a/resources/services/fsx/data_repository_associations.go +++ b/resources/services/fsx/data_repository_associations.go @@ -3,14 +3,15 @@ package fsx import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/fsx" "github.com/aws/aws-sdk-go-v2/service/fsx/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func DataRepositoryAssociations() *schema.Table { diff --git a/resources/services/fsx/data_repository_associations_test.go b/resources/services/fsx/data_repository_associations_test.go index 881532f7..64597f8f 100644 --- a/resources/services/fsx/data_repository_associations_test.go +++ b/resources/services/fsx/data_repository_associations_test.go @@ -3,14 +3,15 @@ package fsx import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/fsx" "github.com/aws/aws-sdk-go-v2/service/fsx/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDataRepoAssociationsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/fsx/data_repository_tasks.go b/resources/services/fsx/data_repository_tasks.go index a2d4ee44..8e956540 100644 --- a/resources/services/fsx/data_repository_tasks.go +++ b/resources/services/fsx/data_repository_tasks.go @@ -3,14 +3,15 @@ package fsx import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/fsx" "github.com/aws/aws-sdk-go-v2/service/fsx/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func DataRepositoryTasks() *schema.Table { diff --git a/resources/services/fsx/data_repository_tasks_test.go b/resources/services/fsx/data_repository_tasks_test.go index 415ee699..6e56d5d5 100644 --- a/resources/services/fsx/data_repository_tasks_test.go +++ b/resources/services/fsx/data_repository_tasks_test.go @@ -3,14 +3,15 @@ package fsx import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/fsx" "github.com/aws/aws-sdk-go-v2/service/fsx/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDataRepoTasksMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/fsx/file_caches.go b/resources/services/fsx/file_caches.go index fcd77a2b..fb167d46 100644 --- a/resources/services/fsx/file_caches.go +++ b/resources/services/fsx/file_caches.go @@ -3,14 +3,15 @@ package fsx import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/fsx" "github.com/aws/aws-sdk-go-v2/service/fsx/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func FileCaches() *schema.Table { diff --git a/resources/services/fsx/file_caches_test.go b/resources/services/fsx/file_caches_test.go index 42a23faf..82248f4c 100644 --- a/resources/services/fsx/file_caches_test.go +++ b/resources/services/fsx/file_caches_test.go @@ -3,13 +3,14 @@ package fsx import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/fsx" "github.com/aws/aws-sdk-go-v2/service/fsx/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildFileCachesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/fsx/file_systems.go b/resources/services/fsx/file_systems.go index d6b6f03f..c7e3e506 100644 --- a/resources/services/fsx/file_systems.go +++ b/resources/services/fsx/file_systems.go @@ -3,14 +3,15 @@ package fsx import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/fsx" "github.com/aws/aws-sdk-go-v2/service/fsx/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func FileSystems() *schema.Table { diff --git a/resources/services/fsx/file_systems_test.go b/resources/services/fsx/file_systems_test.go index f2db8858..b346aaa1 100644 --- a/resources/services/fsx/file_systems_test.go +++ b/resources/services/fsx/file_systems_test.go @@ -3,14 +3,15 @@ package fsx import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/fsx" "github.com/aws/aws-sdk-go-v2/service/fsx/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildFilesystemsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/fsx/snapshots.go b/resources/services/fsx/snapshots.go index a43dd15a..e7fc685c 100644 --- a/resources/services/fsx/snapshots.go +++ b/resources/services/fsx/snapshots.go @@ -3,14 +3,15 @@ package fsx import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/fsx" "github.com/aws/aws-sdk-go-v2/service/fsx/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Snapshots() *schema.Table { diff --git a/resources/services/fsx/snapshots_test.go b/resources/services/fsx/snapshots_test.go index 84f266fb..1ee43c8a 100644 --- a/resources/services/fsx/snapshots_test.go +++ b/resources/services/fsx/snapshots_test.go @@ -3,14 +3,15 @@ package fsx import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/fsx" "github.com/aws/aws-sdk-go-v2/service/fsx/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildSnapshotsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/fsx/storage_virtual_machines.go b/resources/services/fsx/storage_virtual_machines.go index fde5133f..f32ecf2d 100644 --- a/resources/services/fsx/storage_virtual_machines.go +++ b/resources/services/fsx/storage_virtual_machines.go @@ -3,14 +3,15 @@ package fsx import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/fsx" "github.com/aws/aws-sdk-go-v2/service/fsx/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func StorageVirtualMachines() *schema.Table { diff --git a/resources/services/fsx/storage_virtual_machines_test.go b/resources/services/fsx/storage_virtual_machines_test.go index a069eb4e..13376c86 100644 --- a/resources/services/fsx/storage_virtual_machines_test.go +++ b/resources/services/fsx/storage_virtual_machines_test.go @@ -3,14 +3,15 @@ package fsx import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/fsx" "github.com/aws/aws-sdk-go-v2/service/fsx/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildStorageVmsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/fsx/volumes.go b/resources/services/fsx/volumes.go index 2fab91d5..cc51f073 100644 --- a/resources/services/fsx/volumes.go +++ b/resources/services/fsx/volumes.go @@ -3,14 +3,15 @@ package fsx import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/fsx" "github.com/aws/aws-sdk-go-v2/service/fsx/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Volumes() *schema.Table { diff --git a/resources/services/fsx/volumes_test.go b/resources/services/fsx/volumes_test.go index 002fda16..8a6d15e5 100644 --- a/resources/services/fsx/volumes_test.go +++ b/resources/services/fsx/volumes_test.go @@ -3,14 +3,15 @@ package fsx import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/fsx" "github.com/aws/aws-sdk-go-v2/service/fsx/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildVolumesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/glacier/vault_access_policies.go b/resources/services/glacier/vault_access_policies.go index ba48b915..28736997 100644 --- a/resources/services/glacier/vault_access_policies.go +++ b/resources/services/glacier/vault_access_policies.go @@ -3,13 +3,14 @@ package glacier import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/glacier" "github.com/aws/aws-sdk-go-v2/service/glacier/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func vaultAccessPolicies() *schema.Table { diff --git a/resources/services/glacier/vault_lock_policies.go b/resources/services/glacier/vault_lock_policies.go index f8b2b9b4..26142cac 100644 --- a/resources/services/glacier/vault_lock_policies.go +++ b/resources/services/glacier/vault_lock_policies.go @@ -3,13 +3,14 @@ package glacier import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/glacier" "github.com/aws/aws-sdk-go-v2/service/glacier/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func vaultLockPolicies() *schema.Table { diff --git a/resources/services/glacier/vault_notifications.go b/resources/services/glacier/vault_notifications.go index a57a494c..965989e4 100644 --- a/resources/services/glacier/vault_notifications.go +++ b/resources/services/glacier/vault_notifications.go @@ -3,12 +3,13 @@ package glacier import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/glacier" "github.com/aws/aws-sdk-go-v2/service/glacier/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func vaultNotifications() *schema.Table { diff --git a/resources/services/glacier/vaults.go b/resources/services/glacier/vaults.go index 6b58d56f..441f8a56 100644 --- a/resources/services/glacier/vaults.go +++ b/resources/services/glacier/vaults.go @@ -3,13 +3,14 @@ package glacier import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/glacier" "github.com/aws/aws-sdk-go-v2/service/glacier/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Vaults() *schema.Table { diff --git a/resources/services/glacier/vaults_mock_test.go b/resources/services/glacier/vaults_mock_test.go index 3f15b044..bca031ee 100644 --- a/resources/services/glacier/vaults_mock_test.go +++ b/resources/services/glacier/vaults_mock_test.go @@ -3,13 +3,14 @@ package glacier import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/glacier" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildVaultsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/glue/connections.go b/resources/services/glue/connections.go index 01514beb..b4654cbc 100644 --- a/resources/services/glue/connections.go +++ b/resources/services/glue/connections.go @@ -3,14 +3,15 @@ package glue import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/glue" "github.com/aws/aws-sdk-go-v2/service/glue/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Connections() *schema.Table { @@ -50,6 +51,7 @@ func fetchGlueConnections(ctx context.Context, meta schema.ClientMeta, _ *schema } return nil } + func resolveGlueConnectionArn(_ context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { cl := meta.(*client.Client) return resource.Set(c.Name, connectionARN(cl, aws.ToString(resource.Item.(types.Connection).Name))) diff --git a/resources/services/glue/connections_test.go b/resources/services/glue/connections_test.go index 9e0a0ca3..b19b5427 100644 --- a/resources/services/glue/connections_test.go +++ b/resources/services/glue/connections_test.go @@ -3,12 +3,13 @@ package glue import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/glue" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildConnections(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/glue/crawlers.go b/resources/services/glue/crawlers.go index 5fbd8969..a4c76b2c 100644 --- a/resources/services/glue/crawlers.go +++ b/resources/services/glue/crawlers.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/glue" "github.com/aws/aws-sdk-go-v2/service/glue/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Crawlers() *schema.Table { diff --git a/resources/services/glue/crawlers_test.go b/resources/services/glue/crawlers_test.go index dd6597cc..5ef96645 100644 --- a/resources/services/glue/crawlers_test.go +++ b/resources/services/glue/crawlers_test.go @@ -3,12 +3,13 @@ package glue import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/glue" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildCrawlers(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/glue/database_table_indexes.go b/resources/services/glue/database_table_indexes.go index 0a9a70b8..c806d332 100644 --- a/resources/services/glue/database_table_indexes.go +++ b/resources/services/glue/database_table_indexes.go @@ -3,12 +3,13 @@ package glue import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/glue" "github.com/aws/aws-sdk-go-v2/service/glue/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func databaseTableIndexes() *schema.Table { diff --git a/resources/services/glue/database_tables.go b/resources/services/glue/database_tables.go index d5d18e55..0b684263 100644 --- a/resources/services/glue/database_tables.go +++ b/resources/services/glue/database_tables.go @@ -3,12 +3,13 @@ package glue import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/glue" "github.com/aws/aws-sdk-go-v2/service/glue/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func databaseTables() *schema.Table { diff --git a/resources/services/glue/databases.go b/resources/services/glue/databases.go index fc80be9f..fd1fca0b 100644 --- a/resources/services/glue/databases.go +++ b/resources/services/glue/databases.go @@ -3,14 +3,15 @@ package glue import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/glue" "github.com/aws/aws-sdk-go-v2/service/glue/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Databases() *schema.Table { @@ -57,6 +58,7 @@ func fetchGlueDatabases(ctx context.Context, meta schema.ClientMeta, _ *schema.R } return nil } + func resolveGlueDatabaseArn(_ context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { cl := meta.(*client.Client) return resource.Set(c.Name, databaseARN(cl, aws.ToString(resource.Item.(types.Database).Name))) diff --git a/resources/services/glue/databases_mock_test.go b/resources/services/glue/databases_mock_test.go index a794bb5d..c68a155d 100644 --- a/resources/services/glue/databases_mock_test.go +++ b/resources/services/glue/databases_mock_test.go @@ -3,12 +3,13 @@ package glue import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/glue" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDatabasesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/glue/dev_endpoints.go b/resources/services/glue/dev_endpoints.go index 32fa117a..2651cd77 100644 --- a/resources/services/glue/dev_endpoints.go +++ b/resources/services/glue/dev_endpoints.go @@ -3,14 +3,15 @@ package glue import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/glue" "github.com/aws/aws-sdk-go-v2/service/glue/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func DevEndpoints() *schema.Table { @@ -56,6 +57,7 @@ func fetchGlueDevEndpoints(ctx context.Context, meta schema.ClientMeta, _ *schem } return nil } + func resolveGlueDevEndpointArn(_ context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { cl := meta.(*client.Client) return resource.Set(c.Name, devEndpointARN(cl, aws.ToString(resource.Item.(types.DevEndpoint).EndpointName))) diff --git a/resources/services/glue/dev_endpoints_test.go b/resources/services/glue/dev_endpoints_test.go index 63b01396..6341e512 100644 --- a/resources/services/glue/dev_endpoints_test.go +++ b/resources/services/glue/dev_endpoints_test.go @@ -3,12 +3,13 @@ package glue import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/glue" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDevEndpointsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/glue/job_runs.go b/resources/services/glue/job_runs.go index a9e33465..786ff6ad 100644 --- a/resources/services/glue/job_runs.go +++ b/resources/services/glue/job_runs.go @@ -3,12 +3,13 @@ package glue import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/glue" "github.com/aws/aws-sdk-go-v2/service/glue/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func jobRuns() *schema.Table { diff --git a/resources/services/glue/jobs.go b/resources/services/glue/jobs.go index 65ae3973..8ab17787 100644 --- a/resources/services/glue/jobs.go +++ b/resources/services/glue/jobs.go @@ -3,14 +3,15 @@ package glue import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/glue" "github.com/aws/aws-sdk-go-v2/service/glue/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Jobs() *schema.Table { diff --git a/resources/services/glue/jobs_test.go b/resources/services/glue/jobs_test.go index f800ac3a..9c4c338b 100644 --- a/resources/services/glue/jobs_test.go +++ b/resources/services/glue/jobs_test.go @@ -3,13 +3,14 @@ package glue import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/glue" "github.com/aws/aws-sdk-go-v2/service/glue/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildJobsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/glue/ml_transform_task_runs.go b/resources/services/glue/ml_transform_task_runs.go index 627c5724..5c2ab79d 100644 --- a/resources/services/glue/ml_transform_task_runs.go +++ b/resources/services/glue/ml_transform_task_runs.go @@ -3,12 +3,13 @@ package glue import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/glue" "github.com/aws/aws-sdk-go-v2/service/glue/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func mlTransformTaskRuns() *schema.Table { diff --git a/resources/services/glue/ml_transforms.go b/resources/services/glue/ml_transforms.go index e3e9c449..f9e9e5bf 100644 --- a/resources/services/glue/ml_transforms.go +++ b/resources/services/glue/ml_transforms.go @@ -3,15 +3,16 @@ package glue import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/glue" "github.com/aws/aws-sdk-go-v2/service/glue/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func MlTransforms() *schema.Table { @@ -63,6 +64,7 @@ func fetchGlueMlTransforms(ctx context.Context, meta schema.ClientMeta, _ *schem } return nil } + func resolveGlueMlTransformArn(_ context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { cl := meta.(*client.Client) return resource.Set(c.Name, mlTransformARN(cl, aws.ToString(resource.Item.(types.MLTransform).TransformId))) diff --git a/resources/services/glue/ml_transforms_test.go b/resources/services/glue/ml_transforms_test.go index 331cc344..2fe52cab 100644 --- a/resources/services/glue/ml_transforms_test.go +++ b/resources/services/glue/ml_transforms_test.go @@ -3,12 +3,13 @@ package glue import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/glue" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildMlTransformsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/glue/registries.go b/resources/services/glue/registries.go index e14262ff..b7ccbe93 100644 --- a/resources/services/glue/registries.go +++ b/resources/services/glue/registries.go @@ -3,13 +3,14 @@ package glue import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/glue" "github.com/aws/aws-sdk-go-v2/service/glue/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Registries() *schema.Table { diff --git a/resources/services/glue/registries_test.go b/resources/services/glue/registries_test.go index 5c78f6e0..0dad4c2c 100644 --- a/resources/services/glue/registries_test.go +++ b/resources/services/glue/registries_test.go @@ -3,14 +3,15 @@ package glue import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/glue" "github.com/aws/aws-sdk-go-v2/service/glue/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRegistriesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/glue/registry_schema_versions.go b/resources/services/glue/registry_schema_versions.go index df6dea25..68c38c6a 100644 --- a/resources/services/glue/registry_schema_versions.go +++ b/resources/services/glue/registry_schema_versions.go @@ -3,14 +3,15 @@ package glue import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/glue" "github.com/aws/aws-sdk-go-v2/service/glue/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func registrySchemaVersions() *schema.Table { diff --git a/resources/services/glue/registry_schemas.go b/resources/services/glue/registry_schemas.go index fe7b9cce..aab47ae3 100644 --- a/resources/services/glue/registry_schemas.go +++ b/resources/services/glue/registry_schemas.go @@ -3,13 +3,14 @@ package glue import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/glue" "github.com/aws/aws-sdk-go-v2/service/glue/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func registrySchemas() *schema.Table { diff --git a/resources/services/glue/security_configurations.go b/resources/services/glue/security_configurations.go index 3321a05f..de8ff5c9 100644 --- a/resources/services/glue/security_configurations.go +++ b/resources/services/glue/security_configurations.go @@ -3,11 +3,12 @@ package glue import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/service/glue" "github.com/aws/aws-sdk-go-v2/service/glue/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func SecurityConfigurations() *schema.Table { diff --git a/resources/services/glue/security_configurations_test.go b/resources/services/glue/security_configurations_test.go index a58f8296..43818c8b 100644 --- a/resources/services/glue/security_configurations_test.go +++ b/resources/services/glue/security_configurations_test.go @@ -3,12 +3,13 @@ package glue import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/glue" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildSecurityConfigurationsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/glue/tags.go b/resources/services/glue/tags.go index f09d7bfc..92f4c697 100644 --- a/resources/services/glue/tags.go +++ b/resources/services/glue/tags.go @@ -3,11 +3,12 @@ package glue import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/glue" - "github.com/cloudquery/plugin-sdk/v4/schema" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) type arnProvider func(cl *client.Client, resource *schema.Resource) string diff --git a/resources/services/glue/triggers.go b/resources/services/glue/triggers.go index 5ef1484c..fc686928 100644 --- a/resources/services/glue/triggers.go +++ b/resources/services/glue/triggers.go @@ -3,14 +3,15 @@ package glue import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/glue" "github.com/aws/aws-sdk-go-v2/service/glue/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Triggers() *schema.Table { diff --git a/resources/services/glue/triggers_test.go b/resources/services/glue/triggers_test.go index 9139a2b7..7e4d2069 100644 --- a/resources/services/glue/triggers_test.go +++ b/resources/services/glue/triggers_test.go @@ -3,14 +3,15 @@ package glue import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/glue" "github.com/aws/aws-sdk-go-v2/service/glue/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildTriggersMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/glue/workflows.go b/resources/services/glue/workflows.go index b536f693..0487960d 100644 --- a/resources/services/glue/workflows.go +++ b/resources/services/glue/workflows.go @@ -3,14 +3,15 @@ package glue import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/glue" "github.com/aws/aws-sdk-go-v2/service/glue/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Workflows() *schema.Table { diff --git a/resources/services/glue/workflows_test.go b/resources/services/glue/workflows_test.go index 5f9b131b..a55a7b22 100644 --- a/resources/services/glue/workflows_test.go +++ b/resources/services/glue/workflows_test.go @@ -3,14 +3,15 @@ package glue import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/glue" "github.com/aws/aws-sdk-go-v2/service/glue/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildWorkflowsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/guardduty/detector_filters.go b/resources/services/guardduty/detector_filters.go index f874dee5..a0e47f8f 100644 --- a/resources/services/guardduty/detector_filters.go +++ b/resources/services/guardduty/detector_filters.go @@ -3,13 +3,14 @@ package guardduty import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/guardduty/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/guardduty" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/guardduty/models" ) func detectorFilters() *schema.Table { diff --git a/resources/services/guardduty/detector_findings.go b/resources/services/guardduty/detector_findings.go index ffe59a3f..8730eb25 100644 --- a/resources/services/guardduty/detector_findings.go +++ b/resources/services/guardduty/detector_findings.go @@ -3,12 +3,13 @@ package guardduty import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/guardduty/models" "github.com/aws/aws-sdk-go-v2/service/guardduty" "github.com/aws/aws-sdk-go-v2/service/guardduty/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/guardduty/models" ) func detectorFindings() *schema.Table { diff --git a/resources/services/guardduty/detector_ipsets.go b/resources/services/guardduty/detector_ipsets.go index cf18a039..f1ab7244 100644 --- a/resources/services/guardduty/detector_ipsets.go +++ b/resources/services/guardduty/detector_ipsets.go @@ -3,12 +3,13 @@ package guardduty import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/guardduty/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/guardduty" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/guardduty/models" ) func detectorIPSets() *schema.Table { diff --git a/resources/services/guardduty/detector_members.go b/resources/services/guardduty/detector_members.go index fe7df2de..9609913b 100644 --- a/resources/services/guardduty/detector_members.go +++ b/resources/services/guardduty/detector_members.go @@ -3,13 +3,14 @@ package guardduty import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/guardduty/models" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/guardduty" "github.com/aws/aws-sdk-go-v2/service/guardduty/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/guardduty/models" ) func detectorMembers() *schema.Table { diff --git a/resources/services/guardduty/detector_publishing_destinations.go b/resources/services/guardduty/detector_publishing_destinations.go index dc8bbf1b..a5d7f043 100644 --- a/resources/services/guardduty/detector_publishing_destinations.go +++ b/resources/services/guardduty/detector_publishing_destinations.go @@ -3,14 +3,15 @@ package guardduty import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/guardduty/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/guardduty" "github.com/aws/aws-sdk-go-v2/service/guardduty/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/guardduty/models" ) func detectorPublishingDestinations() *schema.Table { diff --git a/resources/services/guardduty/detector_threat_intel_sets.go b/resources/services/guardduty/detector_threat_intel_sets.go index 4f812caf..f96dbde9 100644 --- a/resources/services/guardduty/detector_threat_intel_sets.go +++ b/resources/services/guardduty/detector_threat_intel_sets.go @@ -3,13 +3,14 @@ package guardduty import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/guardduty/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/guardduty" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/guardduty/models" ) func detectorThreatIntelSets() *schema.Table { diff --git a/resources/services/guardduty/detectors.go b/resources/services/guardduty/detectors.go index a2948dba..06ed7801 100644 --- a/resources/services/guardduty/detectors.go +++ b/resources/services/guardduty/detectors.go @@ -3,12 +3,13 @@ package guardduty import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/guardduty/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/guardduty" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/guardduty/models" ) func Detectors() *schema.Table { diff --git a/resources/services/guardduty/detectors_mock_test.go b/resources/services/guardduty/detectors_mock_test.go index f24b7bc1..75cf6c1b 100644 --- a/resources/services/guardduty/detectors_mock_test.go +++ b/resources/services/guardduty/detectors_mock_test.go @@ -4,14 +4,15 @@ import ( "testing" "time" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/guardduty" gdTypes "github.com/aws/aws-sdk-go-v2/service/guardduty/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDetectors(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/iam/credential_reports.go b/resources/services/iam/credential_reports.go index f9b66497..b6aef145 100644 --- a/resources/services/iam/credential_reports.go +++ b/resources/services/iam/credential_reports.go @@ -5,15 +5,16 @@ import ( "errors" "time" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/iam/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/aws/smithy-go" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" "github.com/gocarina/gocsv" "github.com/thoas/go-funk" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/iam/models" ) func CredentialReports() *schema.Table { @@ -151,6 +152,7 @@ func fetchIamCredentialReports(ctx context.Context, meta schema.ClientMeta, _ *s } } } + func timestampPathResolver(path string) schema.ColumnResolver { return func(_ context.Context, meta schema.ClientMeta, r *schema.Resource, c schema.Column) error { t := funk.Get(r.Item, path, funk.WithAllowZero()) diff --git a/resources/services/iam/credential_reports_mock_test.go b/resources/services/iam/credential_reports_mock_test.go index d6baee2e..3ded277a 100644 --- a/resources/services/iam/credential_reports_mock_test.go +++ b/resources/services/iam/credential_reports_mock_test.go @@ -5,12 +5,13 @@ import ( "sync" "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/iam/models" "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/golang/mock/gomock" "github.com/rs/zerolog" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/iam/models" ) var exampleReport = `user,arn,user_creation_time,password_enabled,password_last_used,password_last_changed,password_next_rotation,mfa_active,access_key_1_active,access_key_1_last_rotated,access_key_1_last_used_date,access_key_1_last_used_region,access_key_1_last_used_service,access_key_2_active,access_key_2_last_rotated,access_key_2_last_used_date,access_key_2_last_used_region,access_key_2_last_used_service,cert_1_active,cert_1_last_rotated,cert_2_active,cert_2_last_rotated diff --git a/resources/services/iam/group_attached_policies.go b/resources/services/iam/group_attached_policies.go index 6c766bf6..0dca496b 100644 --- a/resources/services/iam/group_attached_policies.go +++ b/resources/services/iam/group_attached_policies.go @@ -3,13 +3,15 @@ package iam import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/ohaws" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/ohaws" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func groupAttachedPolicies() *schema.Table { diff --git a/resources/services/iam/group_policies.go b/resources/services/iam/group_policies.go index 5def4d35..df3d17ac 100644 --- a/resources/services/iam/group_policies.go +++ b/resources/services/iam/group_policies.go @@ -5,13 +5,15 @@ import ( "encoding/json" "net/url" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/ohaws" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/iam" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/ohaws" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func groupPolicies() *schema.Table { diff --git a/resources/services/iam/groups.go b/resources/services/iam/groups.go index d449749c..41a4bbc9 100644 --- a/resources/services/iam/groups.go +++ b/resources/services/iam/groups.go @@ -4,14 +4,16 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/ohaws" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/iam" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/ohaws" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Groups() *schema.Table { diff --git a/resources/services/iam/groups_mock_test.go b/resources/services/iam/groups_mock_test.go index 89be584f..86ac4ffb 100644 --- a/resources/services/iam/groups_mock_test.go +++ b/resources/services/iam/groups_mock_test.go @@ -3,14 +3,15 @@ package iam import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/iam" iamTypes "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildIamGroups(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/iam/instance_profiles.go b/resources/services/iam/instance_profiles.go index 1158827c..634ae1b3 100644 --- a/resources/services/iam/instance_profiles.go +++ b/resources/services/iam/instance_profiles.go @@ -3,13 +3,14 @@ package iam import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func InstanceProfiles() *schema.Table { diff --git a/resources/services/iam/instance_profiles_mock_test.go b/resources/services/iam/instance_profiles_mock_test.go index 721a20ad..a09ae56d 100644 --- a/resources/services/iam/instance_profiles_mock_test.go +++ b/resources/services/iam/instance_profiles_mock_test.go @@ -3,13 +3,14 @@ package iam import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/iam" iamTypes "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildIamInstanceProfiles(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/iam/mfa_devices.go b/resources/services/iam/mfa_devices.go index 809eadd1..7c4de347 100644 --- a/resources/services/iam/mfa_devices.go +++ b/resources/services/iam/mfa_devices.go @@ -3,12 +3,13 @@ package iam import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func mfaDevices() *schema.Table { diff --git a/resources/services/iam/openid_connect_identity_providers.go b/resources/services/iam/openid_connect_identity_providers.go index dc38c79f..f2000fc1 100644 --- a/resources/services/iam/openid_connect_identity_providers.go +++ b/resources/services/iam/openid_connect_identity_providers.go @@ -3,14 +3,15 @@ package iam import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/iam/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/iam/models" ) func OpenidConnectIdentityProviders() *schema.Table { diff --git a/resources/services/iam/openid_connect_identity_providers_mock_test.go b/resources/services/iam/openid_connect_identity_providers_mock_test.go index 43dfc371..47da01ff 100644 --- a/resources/services/iam/openid_connect_identity_providers_mock_test.go +++ b/resources/services/iam/openid_connect_identity_providers_mock_test.go @@ -3,13 +3,14 @@ package iam import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/iam" iamTypes "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildIamOpenIDConnectProviders(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/iam/password_policies.go b/resources/services/iam/password_policies.go index ace6376d..fc1455ab 100644 --- a/resources/services/iam/password_policies.go +++ b/resources/services/iam/password_policies.go @@ -3,13 +3,14 @@ package iam import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/iam/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/iam" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/iam/models" ) func PasswordPolicies() *schema.Table { diff --git a/resources/services/iam/password_policies_mock_test.go b/resources/services/iam/password_policies_mock_test.go index fb2a1472..b7ae89df 100644 --- a/resources/services/iam/password_policies_mock_test.go +++ b/resources/services/iam/password_policies_mock_test.go @@ -3,13 +3,14 @@ package iam import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/iam" iamTypes "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildIamPasswordPolicies(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/iam/policies.go b/resources/services/iam/policies.go index 3029a570..23589b2b 100644 --- a/resources/services/iam/policies.go +++ b/resources/services/iam/policies.go @@ -3,15 +3,16 @@ package iam import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Policies() *schema.Table { diff --git a/resources/services/iam/policies_mock_test.go b/resources/services/iam/policies_mock_test.go index 8278ad7a..da335c0d 100644 --- a/resources/services/iam/policies_mock_test.go +++ b/resources/services/iam/policies_mock_test.go @@ -4,14 +4,15 @@ import ( "testing" "time" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/iam" iamTypes "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildIamPolicies(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/iam/policy_versions.go b/resources/services/iam/policy_versions.go index fab2b537..768d9a4e 100644 --- a/resources/services/iam/policy_versions.go +++ b/resources/services/iam/policy_versions.go @@ -4,14 +4,15 @@ import ( "context" "net/url" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func policyVersions() *schema.Table { @@ -38,6 +39,7 @@ func policyVersions() *schema.Table { }, } } + func fetchPolicyVersion(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { cl := meta.(*client.Client) policy := parent.Item.(types.Policy) diff --git a/resources/services/iam/role_attached_policies.go b/resources/services/iam/role_attached_policies.go index f38061c5..44c0d453 100644 --- a/resources/services/iam/role_attached_policies.go +++ b/resources/services/iam/role_attached_policies.go @@ -3,12 +3,13 @@ package iam import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func roleAttachedPolicies() *schema.Table { diff --git a/resources/services/iam/role_policies.go b/resources/services/iam/role_policies.go index 050a78dc..fd808cd8 100644 --- a/resources/services/iam/role_policies.go +++ b/resources/services/iam/role_policies.go @@ -5,13 +5,14 @@ import ( "encoding/json" "net/url" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func rolePolicies() *schema.Table { diff --git a/resources/services/iam/roles.go b/resources/services/iam/roles.go index 2df60583..8f3e8ba2 100644 --- a/resources/services/iam/roles.go +++ b/resources/services/iam/roles.go @@ -5,13 +5,14 @@ import ( "encoding/json" "net/url" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Roles() *schema.Table { diff --git a/resources/services/iam/roles_mock_test.go b/resources/services/iam/roles_mock_test.go index 78544ce1..491cac82 100644 --- a/resources/services/iam/roles_mock_test.go +++ b/resources/services/iam/roles_mock_test.go @@ -3,13 +3,14 @@ package iam import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/iam" iamTypes "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRoles(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/iam/saml_identity_providers.go b/resources/services/iam/saml_identity_providers.go index 3b944f94..6964f6d3 100644 --- a/resources/services/iam/saml_identity_providers.go +++ b/resources/services/iam/saml_identity_providers.go @@ -3,13 +3,14 @@ package iam import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/iam/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/iam/models" ) func SamlIdentityProviders() *schema.Table { diff --git a/resources/services/iam/saml_identity_providers_mock_test.go b/resources/services/iam/saml_identity_providers_mock_test.go index 29218e23..1ee90235 100644 --- a/resources/services/iam/saml_identity_providers_mock_test.go +++ b/resources/services/iam/saml_identity_providers_mock_test.go @@ -3,13 +3,14 @@ package iam import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/iam" iamTypes "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildIamSAMLProviders(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/iam/server_certificates.go b/resources/services/iam/server_certificates.go index 85ea7294..da9a3e86 100644 --- a/resources/services/iam/server_certificates.go +++ b/resources/services/iam/server_certificates.go @@ -3,12 +3,13 @@ package iam import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ServerCertificates() *schema.Table { diff --git a/resources/services/iam/server_certificates_mock_test.go b/resources/services/iam/server_certificates_mock_test.go index 8195699d..b94b677d 100644 --- a/resources/services/iam/server_certificates_mock_test.go +++ b/resources/services/iam/server_certificates_mock_test.go @@ -3,13 +3,14 @@ package iam import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/iam" iamTypes "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildIamServerCerts(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/iam/signing_certificates.go b/resources/services/iam/signing_certificates.go index 21a15ac3..075ccf31 100644 --- a/resources/services/iam/signing_certificates.go +++ b/resources/services/iam/signing_certificates.go @@ -3,12 +3,13 @@ package iam import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func signingCertificates() *schema.Table { diff --git a/resources/services/iam/ssh_public_keys.go b/resources/services/iam/ssh_public_keys.go index 1b3b1db9..685dcaab 100644 --- a/resources/services/iam/ssh_public_keys.go +++ b/resources/services/iam/ssh_public_keys.go @@ -3,12 +3,13 @@ package iam import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func sshPublicKeys() *schema.Table { @@ -40,6 +41,7 @@ func sshPublicKeys() *schema.Table { }, } } + func fetchIamSshPublicKeys(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { cl := meta.(*client.Client) svc := cl.Services(client.AWSServiceIam).Iam diff --git a/resources/services/iam/user_access_keys.go b/resources/services/iam/user_access_keys.go index e52a57a4..48fd14b0 100644 --- a/resources/services/iam/user_access_keys.go +++ b/resources/services/iam/user_access_keys.go @@ -3,13 +3,14 @@ package iam import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/iam/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/iam/models" ) func userAccessKeys() *schema.Table { diff --git a/resources/services/iam/user_attached_policies.go b/resources/services/iam/user_attached_policies.go index 9c4c13f5..7234d826 100644 --- a/resources/services/iam/user_attached_policies.go +++ b/resources/services/iam/user_attached_policies.go @@ -3,12 +3,13 @@ package iam import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func userAttachedPolicies() *schema.Table { diff --git a/resources/services/iam/user_groups.go b/resources/services/iam/user_groups.go index 46b50884..d0aaa54e 100644 --- a/resources/services/iam/user_groups.go +++ b/resources/services/iam/user_groups.go @@ -3,12 +3,13 @@ package iam import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func userGroups() *schema.Table { @@ -34,6 +35,7 @@ func userGroups() *schema.Table { }, } } + func fetchIamUserGroups(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { var config iam.ListGroupsForUserInput p := parent.Item.(*types.User) diff --git a/resources/services/iam/user_policies.go b/resources/services/iam/user_policies.go index dad75b1d..15554140 100644 --- a/resources/services/iam/user_policies.go +++ b/resources/services/iam/user_policies.go @@ -5,13 +5,14 @@ import ( "encoding/json" "net/url" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func userPolicies() *schema.Table { diff --git a/resources/services/iam/users.go b/resources/services/iam/users.go index f97878df..a8f0d6a3 100644 --- a/resources/services/iam/users.go +++ b/resources/services/iam/users.go @@ -3,14 +3,15 @@ package iam import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Users() *schema.Table { diff --git a/resources/services/iam/users_mock_test.go b/resources/services/iam/users_mock_test.go index 013b6c44..285cb8bd 100644 --- a/resources/services/iam/users_mock_test.go +++ b/resources/services/iam/users_mock_test.go @@ -3,13 +3,14 @@ package iam import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/aws/aws-sdk-go-v2/service/iam/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildIamUsers(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/identitystore/group_memberships.go b/resources/services/identitystore/group_memberships.go index 9fa4d99a..7bd38a22 100644 --- a/resources/services/identitystore/group_memberships.go +++ b/resources/services/identitystore/group_memberships.go @@ -3,14 +3,15 @@ package identitystore import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/identitystore" "github.com/aws/aws-sdk-go-v2/service/identitystore/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func groupMemberships() *schema.Table { diff --git a/resources/services/identitystore/groups.go b/resources/services/identitystore/groups.go index e484fb4a..dc4e94bb 100644 --- a/resources/services/identitystore/groups.go +++ b/resources/services/identitystore/groups.go @@ -3,14 +3,15 @@ package identitystore import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/identitystore" "github.com/aws/aws-sdk-go-v2/service/identitystore/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Groups() *schema.Table { diff --git a/resources/services/identitystore/groups_mock_test.go b/resources/services/identitystore/groups_mock_test.go index 796871b6..a23fc334 100644 --- a/resources/services/identitystore/groups_mock_test.go +++ b/resources/services/identitystore/groups_mock_test.go @@ -3,15 +3,16 @@ package identitystore import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/identitystore" iTypes "github.com/aws/aws-sdk-go-v2/service/identitystore/types" "github.com/aws/aws-sdk-go-v2/service/ssoadmin" "github.com/aws/aws-sdk-go-v2/service/ssoadmin/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildGroups(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/identitystore/instance_fetch.go b/resources/services/identitystore/instance_fetch.go index d68b3e78..1da9cdcd 100644 --- a/resources/services/identitystore/instance_fetch.go +++ b/resources/services/identitystore/instance_fetch.go @@ -3,10 +3,11 @@ package identitystore import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/service/ssoadmin" "github.com/aws/aws-sdk-go-v2/service/ssoadmin/types" - "github.com/cloudquery/plugin-sdk/v4/schema" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" ) func getIamInstances(ctx context.Context, meta schema.ClientMeta) ([]types.InstanceMetadata, error) { diff --git a/resources/services/identitystore/users.go b/resources/services/identitystore/users.go index ff2af620..8f39274f 100644 --- a/resources/services/identitystore/users.go +++ b/resources/services/identitystore/users.go @@ -3,14 +3,15 @@ package identitystore import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/identitystore" "github.com/aws/aws-sdk-go-v2/service/identitystore/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Users() *schema.Table { diff --git a/resources/services/identitystore/users_mock_test.go b/resources/services/identitystore/users_mock_test.go index f024dd39..9440e340 100644 --- a/resources/services/identitystore/users_mock_test.go +++ b/resources/services/identitystore/users_mock_test.go @@ -3,15 +3,16 @@ package identitystore import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/identitystore" iTypes "github.com/aws/aws-sdk-go-v2/service/identitystore/types" "github.com/aws/aws-sdk-go-v2/service/ssoadmin" "github.com/aws/aws-sdk-go-v2/service/ssoadmin/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildUsers(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/inspector/findings.go b/resources/services/inspector/findings.go index 549df4ee..5f437eaf 100644 --- a/resources/services/inspector/findings.go +++ b/resources/services/inspector/findings.go @@ -3,14 +3,15 @@ package inspector import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/inspector" "github.com/aws/aws-sdk-go-v2/service/inspector/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Findings() *schema.Table { diff --git a/resources/services/inspector/findings_mock_test.go b/resources/services/inspector/findings_mock_test.go index eb299ccc..72e0426b 100644 --- a/resources/services/inspector/findings_mock_test.go +++ b/resources/services/inspector/findings_mock_test.go @@ -3,14 +3,15 @@ package inspector import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/inspector" "github.com/aws/aws-sdk-go-v2/service/inspector/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildInspectorFindings(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/inspector2/findings.go b/resources/services/inspector2/findings.go index bddd0ec2..48cf35b3 100644 --- a/resources/services/inspector2/findings.go +++ b/resources/services/inspector2/findings.go @@ -3,13 +3,14 @@ package inspector2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/inspector2" "github.com/aws/aws-sdk-go-v2/service/inspector2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Findings() *schema.Table { diff --git a/resources/services/inspector2/findings_mock_test.go b/resources/services/inspector2/findings_mock_test.go index 6c62015c..9c4d56fe 100644 --- a/resources/services/inspector2/findings_mock_test.go +++ b/resources/services/inspector2/findings_mock_test.go @@ -3,13 +3,14 @@ package inspector2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/inspector2" "github.com/aws/aws-sdk-go-v2/service/inspector2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildFindings(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/iot/billing_groups.go b/resources/services/iot/billing_groups.go index 9f9e9e86..98e17d76 100644 --- a/resources/services/iot/billing_groups.go +++ b/resources/services/iot/billing_groups.go @@ -3,14 +3,15 @@ package iot import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/iot" "github.com/aws/aws-sdk-go-v2/service/iot/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func BillingGroups() *schema.Table { @@ -104,6 +105,7 @@ func resolveIotBillingGroupThingsInGroup(ctx context.Context, meta schema.Client } return resource.Set(c.Name, things) } + func resolveIotBillingGroupTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { i := resource.Item.(*iot.DescribeBillingGroupOutput) cl := meta.(*client.Client) diff --git a/resources/services/iot/billing_groups_mock_test.go b/resources/services/iot/billing_groups_mock_test.go index 0c4d3f4e..040f8926 100644 --- a/resources/services/iot/billing_groups_mock_test.go +++ b/resources/services/iot/billing_groups_mock_test.go @@ -3,12 +3,13 @@ package iot import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/iot" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildIotBillingGroupsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/iot/ca_certificates.go b/resources/services/iot/ca_certificates.go index a33690d5..da17a70f 100644 --- a/resources/services/iot/ca_certificates.go +++ b/resources/services/iot/ca_certificates.go @@ -3,13 +3,14 @@ package iot import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/iot" "github.com/aws/aws-sdk-go-v2/service/iot/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func CaCertificates() *schema.Table { diff --git a/resources/services/iot/ca_certificates_mock_test.go b/resources/services/iot/ca_certificates_mock_test.go index 4bb3537d..6081382a 100644 --- a/resources/services/iot/ca_certificates_mock_test.go +++ b/resources/services/iot/ca_certificates_mock_test.go @@ -3,12 +3,13 @@ package iot import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/iot" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildIotCaCertificatesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/iot/certificates.go b/resources/services/iot/certificates.go index f2c26380..6a393f10 100644 --- a/resources/services/iot/certificates.go +++ b/resources/services/iot/certificates.go @@ -3,13 +3,14 @@ package iot import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/iot" "github.com/aws/aws-sdk-go-v2/service/iot/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Certificates() *schema.Table { @@ -39,6 +40,7 @@ func Certificates() *schema.Table { }, } } + func fetchIotCertificates(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { cl := meta.(*client.Client) svc := cl.Services(client.AWSServiceIot).Iot diff --git a/resources/services/iot/certificates_mock_test.go b/resources/services/iot/certificates_mock_test.go index ca56b759..f22a664e 100644 --- a/resources/services/iot/certificates_mock_test.go +++ b/resources/services/iot/certificates_mock_test.go @@ -3,12 +3,13 @@ package iot import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/iot" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildIotCertificatesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/iot/jobs.go b/resources/services/iot/jobs.go index 94222b75..29b62de0 100644 --- a/resources/services/iot/jobs.go +++ b/resources/services/iot/jobs.go @@ -3,14 +3,15 @@ package iot import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/iot" "github.com/aws/aws-sdk-go-v2/service/iot/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Jobs() *schema.Table { diff --git a/resources/services/iot/jobs_mock_test.go b/resources/services/iot/jobs_mock_test.go index f4f8c866..fac6721e 100644 --- a/resources/services/iot/jobs_mock_test.go +++ b/resources/services/iot/jobs_mock_test.go @@ -3,12 +3,13 @@ package iot import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/iot" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildIotJobs(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/iot/policies.go b/resources/services/iot/policies.go index 450d96ca..54f84872 100644 --- a/resources/services/iot/policies.go +++ b/resources/services/iot/policies.go @@ -3,14 +3,15 @@ package iot import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/iot" "github.com/aws/aws-sdk-go-v2/service/iot/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Policies() *schema.Table { diff --git a/resources/services/iot/policies_mock_test.go b/resources/services/iot/policies_mock_test.go index 74fd672c..cc8ccc20 100644 --- a/resources/services/iot/policies_mock_test.go +++ b/resources/services/iot/policies_mock_test.go @@ -3,12 +3,13 @@ package iot import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/iot" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildIotPolicies(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/iot/security_profiles.go b/resources/services/iot/security_profiles.go index 67f80411..a816899b 100644 --- a/resources/services/iot/security_profiles.go +++ b/resources/services/iot/security_profiles.go @@ -3,14 +3,15 @@ package iot import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/iot" "github.com/aws/aws-sdk-go-v2/service/iot/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func SecurityProfiles() *schema.Table { @@ -106,6 +107,7 @@ func ResolveIotSecurityProfileTargets(ctx context.Context, meta schema.ClientMet } return resource.Set(c.Name, targets) } + func ResolveIotSecurityProfileTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { i := resource.Item.(*iot.DescribeSecurityProfileOutput) cl := meta.(*client.Client) diff --git a/resources/services/iot/security_profiles_mock_test.go b/resources/services/iot/security_profiles_mock_test.go index 68e14618..2ad21fad 100644 --- a/resources/services/iot/security_profiles_mock_test.go +++ b/resources/services/iot/security_profiles_mock_test.go @@ -3,12 +3,13 @@ package iot import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/iot" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildIotSecurityProfilesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/iot/streams.go b/resources/services/iot/streams.go index e0327fe8..19ffed19 100644 --- a/resources/services/iot/streams.go +++ b/resources/services/iot/streams.go @@ -3,13 +3,14 @@ package iot import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/iot" "github.com/aws/aws-sdk-go-v2/service/iot/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Streams() *schema.Table { diff --git a/resources/services/iot/streams_mock_test.go b/resources/services/iot/streams_mock_test.go index a77b36ef..4147361c 100644 --- a/resources/services/iot/streams_mock_test.go +++ b/resources/services/iot/streams_mock_test.go @@ -3,12 +3,13 @@ package iot import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/iot" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildIotStreamsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/iot/tags.go b/resources/services/iot/tags.go index a3fcd65a..8221cc71 100644 --- a/resources/services/iot/tags.go +++ b/resources/services/iot/tags.go @@ -3,10 +3,11 @@ package iot import ( "context" + "github.com/aws/aws-sdk-go-v2/service/iot" + "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/OpsHelmInc/cloudquery/v2/client/services" - "github.com/aws/aws-sdk-go-v2/service/iot" - "github.com/cloudquery/plugin-sdk/v4/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" ) func resolveIotTags(ctx context.Context, meta schema.ClientMeta, svc services.IotClient, resource *schema.Resource, c schema.Column, resourceArn *string) error { diff --git a/resources/services/iot/thing_groups.go b/resources/services/iot/thing_groups.go index 1cec87be..6ca46c6d 100644 --- a/resources/services/iot/thing_groups.go +++ b/resources/services/iot/thing_groups.go @@ -3,14 +3,15 @@ package iot import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/iot" "github.com/aws/aws-sdk-go-v2/service/iot/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func ThingGroups() *schema.Table { @@ -110,6 +111,7 @@ func ResolveIotThingGroupThingsInGroup(ctx context.Context, meta schema.ClientMe } return resource.Set(c.Name, things) } + func ResolveIotThingGroupPolicies(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { i := resource.Item.(*iot.DescribeThingGroupOutput) cl := meta.(*client.Client) @@ -134,6 +136,7 @@ func ResolveIotThingGroupPolicies(ctx context.Context, meta schema.ClientMeta, r } return resource.Set(c.Name, policies) } + func ResolveIotThingGroupTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { i := resource.Item.(*iot.DescribeThingGroupOutput) cl := meta.(*client.Client) diff --git a/resources/services/iot/thing_groups_mock_test.go b/resources/services/iot/thing_groups_mock_test.go index 402b526b..d9af5e3e 100644 --- a/resources/services/iot/thing_groups_mock_test.go +++ b/resources/services/iot/thing_groups_mock_test.go @@ -3,12 +3,13 @@ package iot import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/iot" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildIotThingGroupsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/iot/thing_types.go b/resources/services/iot/thing_types.go index 6f33d92c..d18cd7a7 100644 --- a/resources/services/iot/thing_types.go +++ b/resources/services/iot/thing_types.go @@ -3,14 +3,15 @@ package iot import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/iot" "github.com/aws/aws-sdk-go-v2/service/iot/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func ThingTypes() *schema.Table { @@ -60,6 +61,7 @@ func fetchIotThingTypes(ctx context.Context, meta schema.ClientMeta, parent *sch } return nil } + func ResolveIotThingTypeTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { i := resource.Item.(types.ThingTypeDefinition) cl := meta.(*client.Client) diff --git a/resources/services/iot/thing_types_mock_test.go b/resources/services/iot/thing_types_mock_test.go index f1810400..c90735db 100644 --- a/resources/services/iot/thing_types_mock_test.go +++ b/resources/services/iot/thing_types_mock_test.go @@ -3,12 +3,13 @@ package iot import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/iot" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildIotThingTypesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/iot/things.go b/resources/services/iot/things.go index d9d20e10..6bf98ae4 100644 --- a/resources/services/iot/things.go +++ b/resources/services/iot/things.go @@ -3,13 +3,14 @@ package iot import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/iot" "github.com/aws/aws-sdk-go-v2/service/iot/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Things() *schema.Table { @@ -58,6 +59,7 @@ func fetchIotThings(ctx context.Context, meta schema.ClientMeta, parent *schema. } return nil } + func ResolveIotThingPrincipals(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { i := resource.Item.(types.ThingAttribute) cl := meta.(*client.Client) diff --git a/resources/services/iot/things_mock_test.go b/resources/services/iot/things_mock_test.go index f50e5b17..b188f22a 100644 --- a/resources/services/iot/things_mock_test.go +++ b/resources/services/iot/things_mock_test.go @@ -3,13 +3,14 @@ package iot import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/iot" "github.com/aws/aws-sdk-go-v2/service/iot/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildIotThingsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/iot/topic_rules.go b/resources/services/iot/topic_rules.go index f8fac651..8a9fe11a 100644 --- a/resources/services/iot/topic_rules.go +++ b/resources/services/iot/topic_rules.go @@ -3,14 +3,15 @@ package iot import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/iot" "github.com/aws/aws-sdk-go-v2/service/iot/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func TopicRules() *schema.Table { diff --git a/resources/services/iot/topic_rules_mock_test.go b/resources/services/iot/topic_rules_mock_test.go index 6a95797a..74210710 100644 --- a/resources/services/iot/topic_rules_mock_test.go +++ b/resources/services/iot/topic_rules_mock_test.go @@ -3,13 +3,14 @@ package iot import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/iot" "github.com/aws/aws-sdk-go-v2/service/iot/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildIotTopicRules(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/kafka/cluster_operations.go b/resources/services/kafka/cluster_operations.go index 9359cd2e..51681632 100644 --- a/resources/services/kafka/cluster_operations.go +++ b/resources/services/kafka/cluster_operations.go @@ -3,13 +3,14 @@ package kafka import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/kafka" "github.com/aws/aws-sdk-go-v2/service/kafka/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func clusterOperations() *schema.Table { diff --git a/resources/services/kafka/cluster_operations_mock_test.go b/resources/services/kafka/cluster_operations_mock_test.go index 434d6d70..ff0c55da 100644 --- a/resources/services/kafka/cluster_operations_mock_test.go +++ b/resources/services/kafka/cluster_operations_mock_test.go @@ -3,12 +3,13 @@ package kafka import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/kafka" "github.com/aws/aws-sdk-go-v2/service/kafka/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildKafkaClusterOperationsMock(t *testing.T, m *mocks.MockKafkaClient) { diff --git a/resources/services/kafka/clusters.go b/resources/services/kafka/clusters.go index 42d9a645..3123b591 100644 --- a/resources/services/kafka/clusters.go +++ b/resources/services/kafka/clusters.go @@ -3,12 +3,13 @@ package kafka import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/kafka" "github.com/aws/aws-sdk-go-v2/service/kafka/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Clusters() *schema.Table { diff --git a/resources/services/kafka/clusters_mock_test.go b/resources/services/kafka/clusters_mock_test.go index 26f885e1..74b9454a 100644 --- a/resources/services/kafka/clusters_mock_test.go +++ b/resources/services/kafka/clusters_mock_test.go @@ -3,13 +3,14 @@ package kafka import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/kafka" "github.com/aws/aws-sdk-go-v2/service/kafka/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildKafkaClustersMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -37,6 +38,7 @@ func buildKafkaClustersMock(t *testing.T, ctrl *gomock.Controller) client.Servic Kafka: m, } } + func TestKafkaClusters(t *testing.T) { client.AwsMockTestHelper(t, Clusters(), buildKafkaClustersMock, client.TestOptions{ SkipEmptyCheckColumns: map[string][]string{"aws_kafka_cluster_operations": {"tags"}}, diff --git a/resources/services/kafka/configurations.go b/resources/services/kafka/configurations.go index a9672df6..b682dacb 100644 --- a/resources/services/kafka/configurations.go +++ b/resources/services/kafka/configurations.go @@ -3,12 +3,13 @@ package kafka import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/kafka" "github.com/aws/aws-sdk-go-v2/service/kafka/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Configurations() *schema.Table { diff --git a/resources/services/kafka/configurations_mock_test.go b/resources/services/kafka/configurations_mock_test.go index d6f25266..c0199ea9 100644 --- a/resources/services/kafka/configurations_mock_test.go +++ b/resources/services/kafka/configurations_mock_test.go @@ -3,13 +3,14 @@ package kafka import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/kafka" "github.com/aws/aws-sdk-go-v2/service/kafka/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildKafkaConfigurationsMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -29,6 +30,7 @@ func buildKafkaConfigurationsMock(t *testing.T, ctrl *gomock.Controller) client. Kafka: m, } } + func TestKafkaConfigurations(t *testing.T) { client.AwsMockTestHelper(t, Configurations(), buildKafkaConfigurationsMock, client.TestOptions{}) } diff --git a/resources/services/kafka/helpers.go b/resources/services/kafka/helpers.go index 215a9c57..ceb31f62 100644 --- a/resources/services/kafka/helpers.go +++ b/resources/services/kafka/helpers.go @@ -3,12 +3,13 @@ package kafka import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/kafka" "github.com/aws/aws-sdk-go-v2/service/kafka/types" - "github.com/cloudquery/plugin-sdk/v4/schema" "github.com/thoas/go-funk" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" ) func getListNodesInput(parent *schema.Resource) kafka.ListNodesInput { diff --git a/resources/services/kafka/nodes.go b/resources/services/kafka/nodes.go index 93b22288..4b95dcbd 100644 --- a/resources/services/kafka/nodes.go +++ b/resources/services/kafka/nodes.go @@ -3,12 +3,13 @@ package kafka import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/kafka" "github.com/aws/aws-sdk-go-v2/service/kafka/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func nodes() *schema.Table { diff --git a/resources/services/kafka/nodes_mock_test.go b/resources/services/kafka/nodes_mock_test.go index 698eb4ef..bcd3df0c 100644 --- a/resources/services/kafka/nodes_mock_test.go +++ b/resources/services/kafka/nodes_mock_test.go @@ -3,12 +3,13 @@ package kafka import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/kafka" "github.com/aws/aws-sdk-go-v2/service/kafka/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildKafkaNodesMock(t *testing.T, m *mocks.MockKafkaClient) { diff --git a/resources/services/kinesis/streams.go b/resources/services/kinesis/streams.go index d583d012..8e5db674 100644 --- a/resources/services/kinesis/streams.go +++ b/resources/services/kinesis/streams.go @@ -3,14 +3,15 @@ package kinesis import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/kinesis" "github.com/aws/aws-sdk-go-v2/service/kinesis/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Streams() *schema.Table { diff --git a/resources/services/kinesis/streams_mock_test.go b/resources/services/kinesis/streams_mock_test.go index 08d62529..247893ed 100644 --- a/resources/services/kinesis/streams_mock_test.go +++ b/resources/services/kinesis/streams_mock_test.go @@ -4,14 +4,15 @@ import ( "testing" "time" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/kinesis" "github.com/aws/aws-sdk-go-v2/service/kinesis/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) type customKinesisClient struct { diff --git a/resources/services/kms/aliases.go b/resources/services/kms/aliases.go index a16436a6..a11d0744 100644 --- a/resources/services/kms/aliases.go +++ b/resources/services/kms/aliases.go @@ -3,12 +3,13 @@ package kms import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/kms" "github.com/aws/aws-sdk-go-v2/service/kms/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Aliases() *schema.Table { diff --git a/resources/services/kms/aliases_mock_test.go b/resources/services/kms/aliases_mock_test.go index e701f729..294cf273 100644 --- a/resources/services/kms/aliases_mock_test.go +++ b/resources/services/kms/aliases_mock_test.go @@ -3,12 +3,13 @@ package kms import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/kms" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildKmsAliases(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/kms/key_grants.go b/resources/services/kms/key_grants.go index cde661e9..22effc49 100644 --- a/resources/services/kms/key_grants.go +++ b/resources/services/kms/key_grants.go @@ -3,13 +3,14 @@ package kms import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/kms" "github.com/aws/aws-sdk-go-v2/service/kms/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func keyGrants() *schema.Table { diff --git a/resources/services/kms/key_policies.go b/resources/services/kms/key_policies.go index 291f1279..f0506ce2 100644 --- a/resources/services/kms/key_policies.go +++ b/resources/services/kms/key_policies.go @@ -3,14 +3,15 @@ package kms import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/kms" "github.com/aws/aws-sdk-go-v2/service/kms/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) type KeyPolicy struct { diff --git a/resources/services/kms/keys.go b/resources/services/kms/keys.go index 0ccb6fef..a6902b2a 100644 --- a/resources/services/kms/keys.go +++ b/resources/services/kms/keys.go @@ -3,14 +3,15 @@ package kms import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/kms" "github.com/aws/aws-sdk-go-v2/service/kms/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Keys() *schema.Table { diff --git a/resources/services/kms/keys_mock_test.go b/resources/services/kms/keys_mock_test.go index 7e40bc93..e84560b6 100644 --- a/resources/services/kms/keys_mock_test.go +++ b/resources/services/kms/keys_mock_test.go @@ -3,15 +3,16 @@ package kms import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/kms" "github.com/aws/aws-sdk-go-v2/service/kms/types" "github.com/aws/smithy-go" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildKmsKeys(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/lambda/function_aliases.go b/resources/services/lambda/function_aliases.go index 2cac91c8..7fc0a811 100644 --- a/resources/services/lambda/function_aliases.go +++ b/resources/services/lambda/function_aliases.go @@ -3,12 +3,13 @@ package lambda import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/lambda" "github.com/aws/aws-sdk-go-v2/service/lambda/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func functionAliases() *schema.Table { diff --git a/resources/services/lambda/function_concurrency_configs.go b/resources/services/lambda/function_concurrency_configs.go index 3778d9ba..5d8ca425 100644 --- a/resources/services/lambda/function_concurrency_configs.go +++ b/resources/services/lambda/function_concurrency_configs.go @@ -3,11 +3,12 @@ package lambda import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/service/lambda" "github.com/aws/aws-sdk-go-v2/service/lambda/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func functionConcurrencyConfigs() *schema.Table { diff --git a/resources/services/lambda/function_event_invoke_configs.go b/resources/services/lambda/function_event_invoke_configs.go index 5a6cfce0..d2b5a07f 100644 --- a/resources/services/lambda/function_event_invoke_configs.go +++ b/resources/services/lambda/function_event_invoke_configs.go @@ -3,11 +3,12 @@ package lambda import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/service/lambda" "github.com/aws/aws-sdk-go-v2/service/lambda/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func functionEventInvokeConfigs() *schema.Table { @@ -25,6 +26,7 @@ func functionEventInvokeConfigs() *schema.Table { }, } } + func fetchLambdaFunctionEventInvokeConfigs(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { p := parent.Item.(*lambda.GetFunctionOutput) if p.Configuration == nil { diff --git a/resources/services/lambda/function_event_source_mappings.go b/resources/services/lambda/function_event_source_mappings.go index 98e2ad6d..4dc297c8 100644 --- a/resources/services/lambda/function_event_source_mappings.go +++ b/resources/services/lambda/function_event_source_mappings.go @@ -3,12 +3,13 @@ package lambda import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/service/lambda" "github.com/aws/aws-sdk-go-v2/service/lambda/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - cqtypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + cqtypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func functionEventSourceMappings() *schema.Table { diff --git a/resources/services/lambda/function_url_configs.go b/resources/services/lambda/function_url_configs.go index 2d460daf..187519f6 100644 --- a/resources/services/lambda/function_url_configs.go +++ b/resources/services/lambda/function_url_configs.go @@ -3,11 +3,12 @@ package lambda import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/service/lambda" "github.com/aws/aws-sdk-go-v2/service/lambda/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func functionURLConfigs() *schema.Table { diff --git a/resources/services/lambda/function_versions.go b/resources/services/lambda/function_versions.go index aaece6bf..2e9cab4f 100644 --- a/resources/services/lambda/function_versions.go +++ b/resources/services/lambda/function_versions.go @@ -3,12 +3,13 @@ package lambda import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/lambda" "github.com/aws/aws-sdk-go-v2/service/lambda/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func functionVersions() *schema.Table { diff --git a/resources/services/lambda/functions.go b/resources/services/lambda/functions.go index fc2649b9..f68c1ef0 100644 --- a/resources/services/lambda/functions.go +++ b/resources/services/lambda/functions.go @@ -4,14 +4,15 @@ import ( "context" "encoding/json" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/lambda" "github.com/aws/aws-sdk-go-v2/service/lambda/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Functions() *schema.Table { diff --git a/resources/services/lambda/functions_mock_test.go b/resources/services/lambda/functions_mock_test.go index 61da2c0b..ac642b7f 100644 --- a/resources/services/lambda/functions_mock_test.go +++ b/resources/services/lambda/functions_mock_test.go @@ -3,16 +3,17 @@ package lambda import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/lambda" "github.com/aws/aws-sdk-go-v2/service/lambda/types" "github.com/aws/smithy-go" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/google/uuid" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildLambdaFunctionsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/lambda/layer_version_policies.go b/resources/services/lambda/layer_version_policies.go index 9540514e..b8056959 100644 --- a/resources/services/lambda/layer_version_policies.go +++ b/resources/services/lambda/layer_version_policies.go @@ -1,11 +1,12 @@ package lambda import ( - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/lambda" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func layerVersionPolicies() *schema.Table { diff --git a/resources/services/lambda/layer_versions.go b/resources/services/lambda/layer_versions.go index b2cca0fd..9f43125f 100644 --- a/resources/services/lambda/layer_versions.go +++ b/resources/services/lambda/layer_versions.go @@ -1,11 +1,12 @@ package lambda import ( - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/lambda/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func layerVersions() *schema.Table { diff --git a/resources/services/lambda/layers.go b/resources/services/lambda/layers.go index c45a8961..8d58f94b 100644 --- a/resources/services/lambda/layers.go +++ b/resources/services/lambda/layers.go @@ -3,13 +3,14 @@ package lambda import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/lambda" "github.com/aws/aws-sdk-go-v2/service/lambda/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Layers() *schema.Table { @@ -55,6 +56,7 @@ func fetchLambdaLayers(ctx context.Context, meta schema.ClientMeta, parent *sche } return nil } + func fetchLambdaLayerVersions(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { p := parent.Item.(types.LayersListItem) cl := meta.(*client.Client) @@ -74,6 +76,7 @@ func fetchLambdaLayerVersions(ctx context.Context, meta schema.ClientMeta, paren } return nil } + func fetchLambdaLayerVersionPolicies(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { p := parent.Item.(types.LayerVersionsListItem) diff --git a/resources/services/lambda/layers_mock_test.go b/resources/services/lambda/layers_mock_test.go index 756e272a..5f469d15 100644 --- a/resources/services/lambda/layers_mock_test.go +++ b/resources/services/lambda/layers_mock_test.go @@ -3,13 +3,14 @@ package lambda import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/lambda" "github.com/aws/aws-sdk-go-v2/service/lambda/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildLambdaLayersMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/lightsail/alarms.go b/resources/services/lightsail/alarms.go index 09482a65..acc3d623 100644 --- a/resources/services/lightsail/alarms.go +++ b/resources/services/lightsail/alarms.go @@ -3,13 +3,14 @@ package lightsail import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/lightsail" "github.com/aws/aws-sdk-go-v2/service/lightsail/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Alarms() *schema.Table { diff --git a/resources/services/lightsail/alarms_mock_test.go b/resources/services/lightsail/alarms_mock_test.go index 62195e23..17fa4673 100644 --- a/resources/services/lightsail/alarms_mock_test.go +++ b/resources/services/lightsail/alarms_mock_test.go @@ -3,12 +3,13 @@ package lightsail import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/lightsail" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildAlarmsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/lightsail/bucket_access_keys.go b/resources/services/lightsail/bucket_access_keys.go index ba34cd8f..134204d7 100644 --- a/resources/services/lightsail/bucket_access_keys.go +++ b/resources/services/lightsail/bucket_access_keys.go @@ -3,12 +3,13 @@ package lightsail import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/lightsail" "github.com/aws/aws-sdk-go-v2/service/lightsail/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func bucketAccessKeys() *schema.Table { diff --git a/resources/services/lightsail/buckets.go b/resources/services/lightsail/buckets.go index 5e8b6504..770ab29d 100644 --- a/resources/services/lightsail/buckets.go +++ b/resources/services/lightsail/buckets.go @@ -3,14 +3,15 @@ package lightsail import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/lightsail" "github.com/aws/aws-sdk-go-v2/service/lightsail/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Buckets() *schema.Table { diff --git a/resources/services/lightsail/buckets_mock_test.go b/resources/services/lightsail/buckets_mock_test.go index 1d2d66c5..203a785c 100644 --- a/resources/services/lightsail/buckets_mock_test.go +++ b/resources/services/lightsail/buckets_mock_test.go @@ -3,12 +3,13 @@ package lightsail import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/lightsail" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildBucketsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/lightsail/certificates.go b/resources/services/lightsail/certificates.go index ef17db4e..c93bdc8e 100644 --- a/resources/services/lightsail/certificates.go +++ b/resources/services/lightsail/certificates.go @@ -3,13 +3,14 @@ package lightsail import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/lightsail" "github.com/aws/aws-sdk-go-v2/service/lightsail/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Certificates() *schema.Table { diff --git a/resources/services/lightsail/certificates_mock_test.go b/resources/services/lightsail/certificates_mock_test.go index fd421006..478e7dd2 100644 --- a/resources/services/lightsail/certificates_mock_test.go +++ b/resources/services/lightsail/certificates_mock_test.go @@ -3,12 +3,13 @@ package lightsail import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/lightsail" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildCertificatesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/lightsail/container_service_deployments.go b/resources/services/lightsail/container_service_deployments.go index 0d4f75e5..ff66fb45 100644 --- a/resources/services/lightsail/container_service_deployments.go +++ b/resources/services/lightsail/container_service_deployments.go @@ -3,12 +3,13 @@ package lightsail import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/lightsail" "github.com/aws/aws-sdk-go-v2/service/lightsail/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func containerServiceDeployments() *schema.Table { diff --git a/resources/services/lightsail/container_service_images.go b/resources/services/lightsail/container_service_images.go index 06878d20..89ff1263 100644 --- a/resources/services/lightsail/container_service_images.go +++ b/resources/services/lightsail/container_service_images.go @@ -3,12 +3,13 @@ package lightsail import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/lightsail" "github.com/aws/aws-sdk-go-v2/service/lightsail/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func containerServiceImages() *schema.Table { diff --git a/resources/services/lightsail/container_services.go b/resources/services/lightsail/container_services.go index b50a2aa5..a555bde5 100644 --- a/resources/services/lightsail/container_services.go +++ b/resources/services/lightsail/container_services.go @@ -3,13 +3,14 @@ package lightsail import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/lightsail" "github.com/aws/aws-sdk-go-v2/service/lightsail/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func ContainerServices() *schema.Table { diff --git a/resources/services/lightsail/container_services_mock_test.go b/resources/services/lightsail/container_services_mock_test.go index 5a763f39..1af9c078 100644 --- a/resources/services/lightsail/container_services_mock_test.go +++ b/resources/services/lightsail/container_services_mock_test.go @@ -3,14 +3,15 @@ package lightsail import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/lightsail" "github.com/aws/aws-sdk-go-v2/service/lightsail/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildContainerServicesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/lightsail/database_events.go b/resources/services/lightsail/database_events.go index 3a44def5..1e09abe6 100644 --- a/resources/services/lightsail/database_events.go +++ b/resources/services/lightsail/database_events.go @@ -3,13 +3,14 @@ package lightsail import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/lightsail" "github.com/aws/aws-sdk-go-v2/service/lightsail/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func databaseEvents() *schema.Table { diff --git a/resources/services/lightsail/database_log_events.go b/resources/services/lightsail/database_log_events.go index 2215dcdf..5c22d8dd 100644 --- a/resources/services/lightsail/database_log_events.go +++ b/resources/services/lightsail/database_log_events.go @@ -4,15 +4,16 @@ import ( "context" "time" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/lightsail/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/lightsail" "github.com/aws/aws-sdk-go-v2/service/lightsail/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" "golang.org/x/sync/errgroup" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/lightsail/models" ) func databaseLogEvents() *schema.Table { diff --git a/resources/services/lightsail/database_parameters.go b/resources/services/lightsail/database_parameters.go index 0db7e083..19326f28 100644 --- a/resources/services/lightsail/database_parameters.go +++ b/resources/services/lightsail/database_parameters.go @@ -3,13 +3,14 @@ package lightsail import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/lightsail" "github.com/aws/aws-sdk-go-v2/service/lightsail/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func databaseParameters() *schema.Table { diff --git a/resources/services/lightsail/database_snapshots.go b/resources/services/lightsail/database_snapshots.go index bc8df852..b2fc3d27 100644 --- a/resources/services/lightsail/database_snapshots.go +++ b/resources/services/lightsail/database_snapshots.go @@ -3,14 +3,15 @@ package lightsail import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/lightsail" "github.com/aws/aws-sdk-go-v2/service/lightsail/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func DatabaseSnapshots() *schema.Table { diff --git a/resources/services/lightsail/database_snapshots_mock_test.go b/resources/services/lightsail/database_snapshots_mock_test.go index 9be2c1b0..c773f867 100644 --- a/resources/services/lightsail/database_snapshots_mock_test.go +++ b/resources/services/lightsail/database_snapshots_mock_test.go @@ -3,12 +3,13 @@ package lightsail import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/lightsail" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDatabaseSnapshotsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/lightsail/databases.go b/resources/services/lightsail/databases.go index dcb2faab..b05972e4 100644 --- a/resources/services/lightsail/databases.go +++ b/resources/services/lightsail/databases.go @@ -3,14 +3,15 @@ package lightsail import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/lightsail" "github.com/aws/aws-sdk-go-v2/service/lightsail/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Databases() *schema.Table { diff --git a/resources/services/lightsail/databases_mock_test.go b/resources/services/lightsail/databases_mock_test.go index 81e10123..d47eff7d 100644 --- a/resources/services/lightsail/databases_mock_test.go +++ b/resources/services/lightsail/databases_mock_test.go @@ -3,12 +3,13 @@ package lightsail import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/lightsail" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDatabasesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/lightsail/disk_snapshots.go b/resources/services/lightsail/disk_snapshots.go index 6298f705..649b4f56 100644 --- a/resources/services/lightsail/disk_snapshots.go +++ b/resources/services/lightsail/disk_snapshots.go @@ -3,14 +3,15 @@ package lightsail import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/lightsail" "github.com/aws/aws-sdk-go-v2/service/lightsail/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func diskSnapshots() *schema.Table { diff --git a/resources/services/lightsail/disks.go b/resources/services/lightsail/disks.go index 2d289f94..962e9929 100644 --- a/resources/services/lightsail/disks.go +++ b/resources/services/lightsail/disks.go @@ -3,14 +3,15 @@ package lightsail import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/lightsail" "github.com/aws/aws-sdk-go-v2/service/lightsail/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Disks() *schema.Table { diff --git a/resources/services/lightsail/disks_mock_test.go b/resources/services/lightsail/disks_mock_test.go index e5d76fd8..81b131ce 100644 --- a/resources/services/lightsail/disks_mock_test.go +++ b/resources/services/lightsail/disks_mock_test.go @@ -3,12 +3,13 @@ package lightsail import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/lightsail" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDisks(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/lightsail/distributions.go b/resources/services/lightsail/distributions.go index dff4136f..b2e6e9c9 100644 --- a/resources/services/lightsail/distributions.go +++ b/resources/services/lightsail/distributions.go @@ -3,16 +3,17 @@ package lightsail import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/lightsail/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/lightsail" "github.com/aws/aws-sdk-go-v2/service/lightsail/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" "golang.org/x/sync/errgroup" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/lightsail/models" ) func Distributions() *schema.Table { diff --git a/resources/services/lightsail/distributions_mock_test.go b/resources/services/lightsail/distributions_mock_test.go index 6ea2bce1..8e9dfb7c 100644 --- a/resources/services/lightsail/distributions_mock_test.go +++ b/resources/services/lightsail/distributions_mock_test.go @@ -3,12 +3,13 @@ package lightsail import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/lightsail" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDistributions(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/lightsail/instance_port_states.go b/resources/services/lightsail/instance_port_states.go index 23f4a8ba..f0a477eb 100644 --- a/resources/services/lightsail/instance_port_states.go +++ b/resources/services/lightsail/instance_port_states.go @@ -4,12 +4,13 @@ import ( "context" "strings" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/lightsail" "github.com/aws/aws-sdk-go-v2/service/lightsail/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func instancePortStates() *schema.Table { diff --git a/resources/services/lightsail/instance_snapshots.go b/resources/services/lightsail/instance_snapshots.go index 2a1004d6..9f7107b0 100644 --- a/resources/services/lightsail/instance_snapshots.go +++ b/resources/services/lightsail/instance_snapshots.go @@ -3,14 +3,15 @@ package lightsail import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/lightsail" "github.com/aws/aws-sdk-go-v2/service/lightsail/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func InstanceSnapshots() *schema.Table { diff --git a/resources/services/lightsail/instance_snapshots_mock_test.go b/resources/services/lightsail/instance_snapshots_mock_test.go index fa26623e..5d9d54f8 100644 --- a/resources/services/lightsail/instance_snapshots_mock_test.go +++ b/resources/services/lightsail/instance_snapshots_mock_test.go @@ -3,12 +3,13 @@ package lightsail import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/lightsail" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildInstanceSnapshots(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/lightsail/instances.go b/resources/services/lightsail/instances.go index 5d352296..733ce463 100644 --- a/resources/services/lightsail/instances.go +++ b/resources/services/lightsail/instances.go @@ -3,14 +3,15 @@ package lightsail import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/lightsail" "github.com/aws/aws-sdk-go-v2/service/lightsail/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Instances() *schema.Table { @@ -69,6 +70,7 @@ func fetchLightsailInstances(ctx context.Context, meta schema.ClientMeta, parent } return nil } + func resolveLightsailInstanceAccessDetails(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { r := resource.Item.(types.Instance) cli := meta.(*client.Client) diff --git a/resources/services/lightsail/instances_mock_test.go b/resources/services/lightsail/instances_mock_test.go index 41fc4056..6ff069a8 100644 --- a/resources/services/lightsail/instances_mock_test.go +++ b/resources/services/lightsail/instances_mock_test.go @@ -3,13 +3,14 @@ package lightsail import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/lightsail" "github.com/aws/aws-sdk-go-v2/service/lightsail/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildInstances(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/lightsail/load_balancer_tls_certificates.go b/resources/services/lightsail/load_balancer_tls_certificates.go index 7928c104..42ff091d 100644 --- a/resources/services/lightsail/load_balancer_tls_certificates.go +++ b/resources/services/lightsail/load_balancer_tls_certificates.go @@ -3,13 +3,14 @@ package lightsail import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/lightsail" "github.com/aws/aws-sdk-go-v2/service/lightsail/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func loadBalancerTlsCertificates() *schema.Table { diff --git a/resources/services/lightsail/load_balancers.go b/resources/services/lightsail/load_balancers.go index 18bbcff0..d6a5b9d1 100644 --- a/resources/services/lightsail/load_balancers.go +++ b/resources/services/lightsail/load_balancers.go @@ -3,14 +3,15 @@ package lightsail import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/lightsail" "github.com/aws/aws-sdk-go-v2/service/lightsail/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func LoadBalancers() *schema.Table { diff --git a/resources/services/lightsail/load_balancers_mock_test.go b/resources/services/lightsail/load_balancers_mock_test.go index b8faac53..82980fd2 100644 --- a/resources/services/lightsail/load_balancers_mock_test.go +++ b/resources/services/lightsail/load_balancers_mock_test.go @@ -3,12 +3,13 @@ package lightsail import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/lightsail" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildLoadBalancers(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/lightsail/static_ips.go b/resources/services/lightsail/static_ips.go index 610edb9c..baad7f24 100644 --- a/resources/services/lightsail/static_ips.go +++ b/resources/services/lightsail/static_ips.go @@ -3,13 +3,14 @@ package lightsail import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/lightsail" "github.com/aws/aws-sdk-go-v2/service/lightsail/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func StaticIps() *schema.Table { diff --git a/resources/services/lightsail/static_ips_mock_test.go b/resources/services/lightsail/static_ips_mock_test.go index cd971252..a32fe963 100644 --- a/resources/services/lightsail/static_ips_mock_test.go +++ b/resources/services/lightsail/static_ips_mock_test.go @@ -3,12 +3,13 @@ package lightsail import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/lightsail" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildStaticIps(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/mq/broker_configuration_revisions.go b/resources/services/mq/broker_configuration_revisions.go index d6218050..f8783e75 100644 --- a/resources/services/mq/broker_configuration_revisions.go +++ b/resources/services/mq/broker_configuration_revisions.go @@ -8,15 +8,15 @@ import ( "strconv" "github.com/apache/arrow/go/v16/arrow" - xj "github.com/basgys/goxml2json" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" - - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/mq" "github.com/aws/aws-sdk-go-v2/service/mq/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + xj "github.com/basgys/goxml2json" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) type wrappedBrokerConfigurationRevision struct { diff --git a/resources/services/mq/broker_configurations.go b/resources/services/mq/broker_configurations.go index fb7620a8..f1020c6a 100644 --- a/resources/services/mq/broker_configurations.go +++ b/resources/services/mq/broker_configurations.go @@ -3,11 +3,12 @@ package mq import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/mq" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func brokerConfigurations() *schema.Table { diff --git a/resources/services/mq/broker_users.go b/resources/services/mq/broker_users.go index 20e414c0..41ec8903 100644 --- a/resources/services/mq/broker_users.go +++ b/resources/services/mq/broker_users.go @@ -3,11 +3,12 @@ package mq import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/mq" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func brokerUsers() *schema.Table { diff --git a/resources/services/mq/brokers.go b/resources/services/mq/brokers.go index d55921c8..913e8a5d 100644 --- a/resources/services/mq/brokers.go +++ b/resources/services/mq/brokers.go @@ -3,12 +3,13 @@ package mq import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/mq" "github.com/aws/aws-sdk-go-v2/service/mq/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Brokers() *schema.Table { diff --git a/resources/services/mq/brokers_mock_test.go b/resources/services/mq/brokers_mock_test.go index 9073d08c..ef195209 100644 --- a/resources/services/mq/brokers_mock_test.go +++ b/resources/services/mq/brokers_mock_test.go @@ -3,14 +3,15 @@ package mq import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/mq" "github.com/aws/aws-sdk-go-v2/service/mq/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildMqBrokers(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/mwaa/environments.go b/resources/services/mwaa/environments.go index cd063ba9..10709ad1 100644 --- a/resources/services/mwaa/environments.go +++ b/resources/services/mwaa/environments.go @@ -3,12 +3,13 @@ package mwaa import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/mwaa" "github.com/aws/aws-sdk-go-v2/service/mwaa/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Environments() *schema.Table { diff --git a/resources/services/mwaa/environments_mock_test.go b/resources/services/mwaa/environments_mock_test.go index c941cdb9..dfbb2c95 100644 --- a/resources/services/mwaa/environments_mock_test.go +++ b/resources/services/mwaa/environments_mock_test.go @@ -3,13 +3,14 @@ package mwaa import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/mwaa" "github.com/aws/aws-sdk-go-v2/service/mwaa/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildMwaaEnvironments(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/neptune/cluster_parameter_group_parameters.go b/resources/services/neptune/cluster_parameter_group_parameters.go index 1c9ac20e..26ed32fa 100644 --- a/resources/services/neptune/cluster_parameter_group_parameters.go +++ b/resources/services/neptune/cluster_parameter_group_parameters.go @@ -3,12 +3,13 @@ package neptune import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/neptune" "github.com/aws/aws-sdk-go-v2/service/neptune/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func clusterParameterGroupParameters() *schema.Table { diff --git a/resources/services/neptune/cluster_parameter_groups.go b/resources/services/neptune/cluster_parameter_groups.go index 2de17692..e13ea05e 100644 --- a/resources/services/neptune/cluster_parameter_groups.go +++ b/resources/services/neptune/cluster_parameter_groups.go @@ -3,14 +3,15 @@ package neptune import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/neptune" "github.com/aws/aws-sdk-go-v2/service/neptune/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func ClusterParameterGroups() *schema.Table { diff --git a/resources/services/neptune/cluster_parameter_groups_mock_test.go b/resources/services/neptune/cluster_parameter_groups_mock_test.go index 4e7775b1..ffdc3218 100644 --- a/resources/services/neptune/cluster_parameter_groups_mock_test.go +++ b/resources/services/neptune/cluster_parameter_groups_mock_test.go @@ -3,14 +3,15 @@ package neptune import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/neptune" "github.com/aws/aws-sdk-go-v2/service/neptune/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildNeptuneClusterParameterGroups(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/neptune/cluster_snapshots.go b/resources/services/neptune/cluster_snapshots.go index 9f25fe60..55621ce8 100644 --- a/resources/services/neptune/cluster_snapshots.go +++ b/resources/services/neptune/cluster_snapshots.go @@ -3,14 +3,15 @@ package neptune import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/neptune" "github.com/aws/aws-sdk-go-v2/service/neptune/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func ClusterSnapshots() *schema.Table { diff --git a/resources/services/neptune/cluster_snapshots_mock_test.go b/resources/services/neptune/cluster_snapshots_mock_test.go index 45c833a9..c1d2000c 100644 --- a/resources/services/neptune/cluster_snapshots_mock_test.go +++ b/resources/services/neptune/cluster_snapshots_mock_test.go @@ -3,14 +3,15 @@ package neptune import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/neptune" "github.com/aws/aws-sdk-go-v2/service/neptune/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildNeptuneClientForClusterSnapshots(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/neptune/clusters.go b/resources/services/neptune/clusters.go index 894c22e7..c5444514 100644 --- a/resources/services/neptune/clusters.go +++ b/resources/services/neptune/clusters.go @@ -3,14 +3,15 @@ package neptune import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/neptune" "github.com/aws/aws-sdk-go-v2/service/neptune/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Clusters() *schema.Table { diff --git a/resources/services/neptune/db_parameter_group_db_parameters.go b/resources/services/neptune/db_parameter_group_db_parameters.go index 1f9d4ac4..6f1a87bc 100644 --- a/resources/services/neptune/db_parameter_group_db_parameters.go +++ b/resources/services/neptune/db_parameter_group_db_parameters.go @@ -1,11 +1,12 @@ package neptune import ( - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/neptune/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func dbParameterGroupDbParameters() *schema.Table { diff --git a/resources/services/neptune/db_parameter_groups.go b/resources/services/neptune/db_parameter_groups.go index da134a89..8836cf40 100644 --- a/resources/services/neptune/db_parameter_groups.go +++ b/resources/services/neptune/db_parameter_groups.go @@ -3,14 +3,15 @@ package neptune import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/neptune" "github.com/aws/aws-sdk-go-v2/service/neptune/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func DbParameterGroups() *schema.Table { diff --git a/resources/services/neptune/db_parameter_groups_mock_test.go b/resources/services/neptune/db_parameter_groups_mock_test.go index 7c410944..a12c2cdc 100644 --- a/resources/services/neptune/db_parameter_groups_mock_test.go +++ b/resources/services/neptune/db_parameter_groups_mock_test.go @@ -3,14 +3,15 @@ package neptune import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/neptune" "github.com/aws/aws-sdk-go-v2/service/neptune/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildNeptuneDBParameterGroups(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/neptune/event_subscriptions.go b/resources/services/neptune/event_subscriptions.go index 101f5803..3f9e648e 100644 --- a/resources/services/neptune/event_subscriptions.go +++ b/resources/services/neptune/event_subscriptions.go @@ -3,14 +3,15 @@ package neptune import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/neptune" "github.com/aws/aws-sdk-go-v2/service/neptune/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func EventSubscriptions() *schema.Table { diff --git a/resources/services/neptune/event_subscriptions_mock_test.go b/resources/services/neptune/event_subscriptions_mock_test.go index d51d38f5..b9a92e1c 100644 --- a/resources/services/neptune/event_subscriptions_mock_test.go +++ b/resources/services/neptune/event_subscriptions_mock_test.go @@ -3,14 +3,15 @@ package neptune import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/neptune" "github.com/aws/aws-sdk-go-v2/service/neptune/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildNeptuneEventSubscriptions(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/neptune/global_clusters.go b/resources/services/neptune/global_clusters.go index b731e29a..37e61273 100644 --- a/resources/services/neptune/global_clusters.go +++ b/resources/services/neptune/global_clusters.go @@ -3,12 +3,13 @@ package neptune import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/neptune" "github.com/aws/aws-sdk-go-v2/service/neptune/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func GlobalClusters() *schema.Table { diff --git a/resources/services/neptune/global_clusters_mock_test.go b/resources/services/neptune/global_clusters_mock_test.go index 351d17d1..bea1fda2 100644 --- a/resources/services/neptune/global_clusters_mock_test.go +++ b/resources/services/neptune/global_clusters_mock_test.go @@ -3,13 +3,14 @@ package neptune import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/neptune" "github.com/aws/aws-sdk-go-v2/service/neptune/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildNeptuneGlobalClusters(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/neptune/instances.go b/resources/services/neptune/instances.go index 122e8f30..106f8f69 100644 --- a/resources/services/neptune/instances.go +++ b/resources/services/neptune/instances.go @@ -3,14 +3,15 @@ package neptune import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/neptune" "github.com/aws/aws-sdk-go-v2/service/neptune/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Instances() *schema.Table { diff --git a/resources/services/neptune/mock_test.go b/resources/services/neptune/mock_test.go index 7eb814b3..da61de84 100644 --- a/resources/services/neptune/mock_test.go +++ b/resources/services/neptune/mock_test.go @@ -3,14 +3,15 @@ package neptune import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/neptune" "github.com/aws/aws-sdk-go-v2/service/neptune/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildNeptuneDBClusters(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -90,9 +91,11 @@ func buildNeptuneDBSubnetGroups(t *testing.T, ctrl *gomock.Controller) client.Se func TestNeptuneInstances(t *testing.T) { client.AwsMockTestHelper(t, Instances(), buildNeptuneDBInstances, client.TestOptions{}) } + func TestNeptuneClusters(t *testing.T) { client.AwsMockTestHelper(t, Clusters(), buildNeptuneDBClusters, client.TestOptions{}) } + func TestNeptuneSubnetGroups(t *testing.T) { client.AwsMockTestHelper(t, SubnetGroups(), buildNeptuneDBSubnetGroups, client.TestOptions{}) } diff --git a/resources/services/neptune/subnet_groups.go b/resources/services/neptune/subnet_groups.go index 556afef4..9342d6d0 100644 --- a/resources/services/neptune/subnet_groups.go +++ b/resources/services/neptune/subnet_groups.go @@ -3,14 +3,15 @@ package neptune import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/neptune" "github.com/aws/aws-sdk-go-v2/service/neptune/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func SubnetGroups() *schema.Table { diff --git a/resources/services/networkfirewall/firewall_policies.go b/resources/services/networkfirewall/firewall_policies.go index 5f396f33..732fe652 100644 --- a/resources/services/networkfirewall/firewall_policies.go +++ b/resources/services/networkfirewall/firewall_policies.go @@ -3,14 +3,15 @@ package networkfirewall import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/networkfirewall/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/networkfirewall" "github.com/aws/aws-sdk-go-v2/service/networkfirewall/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/networkfirewall/models" ) func FirewallPolicies() *schema.Table { diff --git a/resources/services/networkfirewall/firewall_policies_mock_test.go b/resources/services/networkfirewall/firewall_policies_mock_test.go index b7691469..bed13fb6 100644 --- a/resources/services/networkfirewall/firewall_policies_mock_test.go +++ b/resources/services/networkfirewall/firewall_policies_mock_test.go @@ -3,13 +3,14 @@ package networkfirewall import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/networkfirewall" "github.com/aws/aws-sdk-go-v2/service/networkfirewall/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildFirewallPoliciesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/networkfirewall/firewalls.go b/resources/services/networkfirewall/firewalls.go index cab0a614..5c6e1b8b 100644 --- a/resources/services/networkfirewall/firewalls.go +++ b/resources/services/networkfirewall/firewalls.go @@ -3,14 +3,15 @@ package networkfirewall import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/networkfirewall/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/networkfirewall" "github.com/aws/aws-sdk-go-v2/service/networkfirewall/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/networkfirewall/models" ) func Firewalls() *schema.Table { diff --git a/resources/services/networkfirewall/firewalls_mock_test.go b/resources/services/networkfirewall/firewalls_mock_test.go index 9ee6bd99..e8340cc6 100644 --- a/resources/services/networkfirewall/firewalls_mock_test.go +++ b/resources/services/networkfirewall/firewalls_mock_test.go @@ -3,13 +3,14 @@ package networkfirewall import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/networkfirewall" "github.com/aws/aws-sdk-go-v2/service/networkfirewall/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildFirewallsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/networkfirewall/models/networkfirewall.go b/resources/services/networkfirewall/models/networkfirewall.go index 852c082f..47e2a360 100644 --- a/resources/services/networkfirewall/models/networkfirewall.go +++ b/resources/services/networkfirewall/models/networkfirewall.go @@ -18,6 +18,7 @@ type FirewallWrapper struct { *types.FirewallStatus *types.Firewall } + type TLSInspectionConfigurationWrapper struct { *types.TLSInspectionConfiguration *types.TLSInspectionConfigurationResponse diff --git a/resources/services/networkfirewall/rule_groups.go b/resources/services/networkfirewall/rule_groups.go index b33d54c8..86dedff9 100644 --- a/resources/services/networkfirewall/rule_groups.go +++ b/resources/services/networkfirewall/rule_groups.go @@ -3,14 +3,15 @@ package networkfirewall import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/networkfirewall/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/networkfirewall" "github.com/aws/aws-sdk-go-v2/service/networkfirewall/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/networkfirewall/models" ) func RuleGroups() *schema.Table { diff --git a/resources/services/networkfirewall/rule_groups_mock_test.go b/resources/services/networkfirewall/rule_groups_mock_test.go index 68f9bff1..fdd12b11 100644 --- a/resources/services/networkfirewall/rule_groups_mock_test.go +++ b/resources/services/networkfirewall/rule_groups_mock_test.go @@ -3,13 +3,14 @@ package networkfirewall import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/networkfirewall" "github.com/aws/aws-sdk-go-v2/service/networkfirewall/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRuleGroupsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/networkfirewall/tls_inspection_configurations.go b/resources/services/networkfirewall/tls_inspection_configurations.go index dd4e58a2..11b90943 100644 --- a/resources/services/networkfirewall/tls_inspection_configurations.go +++ b/resources/services/networkfirewall/tls_inspection_configurations.go @@ -3,14 +3,15 @@ package networkfirewall import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/networkfirewall/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/networkfirewall" "github.com/aws/aws-sdk-go-v2/service/networkfirewall/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/networkfirewall/models" ) func TLSInspectionConfigurations() *schema.Table { diff --git a/resources/services/networkfirewall/tls_inspection_configurations_mock_test.go b/resources/services/networkfirewall/tls_inspection_configurations_mock_test.go index 5b7d0779..52e213db 100644 --- a/resources/services/networkfirewall/tls_inspection_configurations_mock_test.go +++ b/resources/services/networkfirewall/tls_inspection_configurations_mock_test.go @@ -3,13 +3,14 @@ package networkfirewall import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/networkfirewall" "github.com/aws/aws-sdk-go-v2/service/networkfirewall/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildTLSInspectionConfigurationsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/networkmanager/global_networks.go b/resources/services/networkmanager/global_networks.go index 00b56408..eea616ce 100644 --- a/resources/services/networkmanager/global_networks.go +++ b/resources/services/networkmanager/global_networks.go @@ -3,13 +3,14 @@ package networkmanager import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/networkmanager" "github.com/aws/aws-sdk-go-v2/service/networkmanager/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func GlobalNetworks() *schema.Table { diff --git a/resources/services/networkmanager/global_networks_mock_test.go b/resources/services/networkmanager/global_networks_mock_test.go index f1e67bc5..27658a0d 100644 --- a/resources/services/networkmanager/global_networks_mock_test.go +++ b/resources/services/networkmanager/global_networks_mock_test.go @@ -3,13 +3,14 @@ package networkmanager import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/networkmanager" "github.com/aws/aws-sdk-go-v2/service/networkmanager/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildNetworksMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/networkmanager/link.go b/resources/services/networkmanager/link.go index 0b66b3a5..cec221ae 100644 --- a/resources/services/networkmanager/link.go +++ b/resources/services/networkmanager/link.go @@ -3,13 +3,14 @@ package networkmanager import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/networkmanager" "github.com/aws/aws-sdk-go-v2/service/networkmanager/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func links() *schema.Table { diff --git a/resources/services/networkmanager/sites.go b/resources/services/networkmanager/sites.go index db1ae2e6..e2a05d7b 100644 --- a/resources/services/networkmanager/sites.go +++ b/resources/services/networkmanager/sites.go @@ -3,13 +3,14 @@ package networkmanager import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/networkmanager" "github.com/aws/aws-sdk-go-v2/service/networkmanager/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func sites() *schema.Table { diff --git a/resources/services/networkmanager/transit_gateway_registrations.go b/resources/services/networkmanager/transit_gateway_registrations.go index 9da0c7bd..f98744c9 100644 --- a/resources/services/networkmanager/transit_gateway_registrations.go +++ b/resources/services/networkmanager/transit_gateway_registrations.go @@ -3,11 +3,12 @@ package networkmanager import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/service/networkmanager" "github.com/aws/aws-sdk-go-v2/service/networkmanager/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func transitGatewayRegistration() *schema.Table { diff --git a/resources/services/organizations/account_parents.go b/resources/services/organizations/account_parents.go index b65673d2..a89f332d 100644 --- a/resources/services/organizations/account_parents.go +++ b/resources/services/organizations/account_parents.go @@ -3,12 +3,13 @@ package organizations import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/organizations" "github.com/aws/aws-sdk-go-v2/service/organizations/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func organizationalAccountParents() *schema.Table { @@ -36,6 +37,7 @@ The 'request_account_id' column is added to show from where the request was made }, } } + func fetchParents(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { cl := meta.(*client.Client) svc := cl.Services(client.AWSServiceOrganizations).Organizations diff --git a/resources/services/organizations/accounts.go b/resources/services/organizations/accounts.go index ae46f861..a25b8058 100644 --- a/resources/services/organizations/accounts.go +++ b/resources/services/organizations/accounts.go @@ -3,13 +3,14 @@ package organizations import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/organizations" "github.com/aws/aws-sdk-go-v2/service/organizations/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Accounts() *schema.Table { @@ -58,6 +59,7 @@ func fetchOrganizationsAccounts(ctx context.Context, meta schema.ClientMeta, _ * } return nil } + func resolveAccountTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, _ schema.Column) error { cl := meta.(*client.Client) account := resource.Item.(types.Account) diff --git a/resources/services/organizations/accounts_mock_test.go b/resources/services/organizations/accounts_mock_test.go index b0f5d103..dc264a6b 100644 --- a/resources/services/organizations/accounts_mock_test.go +++ b/resources/services/organizations/accounts_mock_test.go @@ -3,13 +3,14 @@ package organizations import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/organizations" organizationsTypes "github.com/aws/aws-sdk-go-v2/service/organizations/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildOrganizationsAccounts(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/organizations/delegated_admins.go b/resources/services/organizations/delegated_admins.go index c87485ca..c8deafb5 100644 --- a/resources/services/organizations/delegated_admins.go +++ b/resources/services/organizations/delegated_admins.go @@ -3,12 +3,13 @@ package organizations import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/organizations" "github.com/aws/aws-sdk-go-v2/service/organizations/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func DelegatedAdministrators() *schema.Table { @@ -34,6 +35,7 @@ func DelegatedAdministrators() *schema.Table { }, } } + func fetchOrganizationsDelegatedAdmins(ctx context.Context, meta schema.ClientMeta, _ *schema.Resource, res chan<- any) error { cl := meta.(*client.Client) svc := cl.Services(client.AWSServiceOrganizations).Organizations diff --git a/resources/services/organizations/delegated_admins_mock_test.go b/resources/services/organizations/delegated_admins_mock_test.go index f921ffb1..acb18f2f 100644 --- a/resources/services/organizations/delegated_admins_mock_test.go +++ b/resources/services/organizations/delegated_admins_mock_test.go @@ -3,13 +3,14 @@ package organizations import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/organizations" "github.com/aws/aws-sdk-go-v2/service/organizations/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDelegatedAdministrators(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/organizations/delegated_services.go b/resources/services/organizations/delegated_services.go index f32cbf90..0ebe490d 100644 --- a/resources/services/organizations/delegated_services.go +++ b/resources/services/organizations/delegated_services.go @@ -3,11 +3,12 @@ package organizations import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/service/organizations" "github.com/aws/aws-sdk-go-v2/service/organizations/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func delegatedServices() *schema.Table { @@ -22,6 +23,7 @@ func delegatedServices() *schema.Table { }, } } + func fetchOrganizationsDelegatedServices(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { cl := meta.(*client.Client) svc := cl.Services(client.AWSServiceOrganizations).Organizations diff --git a/resources/services/organizations/organizational_unit_parents.go b/resources/services/organizations/organizational_unit_parents.go index 42a85e99..9ee1ad62 100644 --- a/resources/services/organizations/organizational_unit_parents.go +++ b/resources/services/organizations/organizational_unit_parents.go @@ -3,12 +3,13 @@ package organizations import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/organizations" "github.com/aws/aws-sdk-go-v2/service/organizations/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func organizationalUnitParents() *schema.Table { @@ -36,6 +37,7 @@ The 'request_account_id' column is added to show from where the request was made }, } } + func fetchOUParents(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { cl := meta.(*client.Client) svc := cl.Services(client.AWSServiceOrganizations).Organizations diff --git a/resources/services/organizations/organizational_units.go b/resources/services/organizations/organizational_units.go index 2c0f2b60..268e6399 100644 --- a/resources/services/organizations/organizational_units.go +++ b/resources/services/organizations/organizational_units.go @@ -3,14 +3,15 @@ package organizations import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/services" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/organizations" "github.com/aws/aws-sdk-go-v2/service/organizations/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/services" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func OrganizationalUnits() *schema.Table { diff --git a/resources/services/organizations/organizational_units_mock_test.go b/resources/services/organizations/organizational_units_mock_test.go index 3a147482..13cf1ee2 100644 --- a/resources/services/organizations/organizational_units_mock_test.go +++ b/resources/services/organizations/organizational_units_mock_test.go @@ -3,13 +3,14 @@ package organizations import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/organizations" "github.com/aws/aws-sdk-go-v2/service/organizations/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildOrganizationalUnits(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/organizations/organizations.go b/resources/services/organizations/organizations.go index 86b57e38..27037784 100644 --- a/resources/services/organizations/organizations.go +++ b/resources/services/organizations/organizations.go @@ -3,12 +3,13 @@ package organizations import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/organizations" "github.com/aws/aws-sdk-go-v2/service/organizations/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Organizations() *schema.Table { @@ -37,6 +38,7 @@ The 'request_account_id' column is added to show from where the request was made }, } } + func fetchOrganizationsOrganizations(ctx context.Context, meta schema.ClientMeta, _ *schema.Resource, res chan<- any) error { cl := meta.(*client.Client) svc := cl.Services(client.AWSServiceOrganizations).Organizations diff --git a/resources/services/organizations/organizations_mock_test.go b/resources/services/organizations/organizations_mock_test.go index 4a65aea0..c73cb530 100644 --- a/resources/services/organizations/organizations_mock_test.go +++ b/resources/services/organizations/organizations_mock_test.go @@ -3,12 +3,13 @@ package organizations import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/organizations" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildOrganizations(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/organizations/policies.go b/resources/services/organizations/policies.go index 4cbce882..ca4ffc5e 100644 --- a/resources/services/organizations/policies.go +++ b/resources/services/organizations/policies.go @@ -3,13 +3,14 @@ package organizations import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/organizations" "github.com/aws/aws-sdk-go-v2/service/organizations/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Policies() *schema.Table { diff --git a/resources/services/organizations/resource_policies.go b/resources/services/organizations/resource_policies.go index a885392c..fd56006e 100644 --- a/resources/services/organizations/resource_policies.go +++ b/resources/services/organizations/resource_policies.go @@ -3,12 +3,13 @@ package organizations import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/organizations" "github.com/aws/aws-sdk-go-v2/service/organizations/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ResourcePolicies() *schema.Table { diff --git a/resources/services/organizations/resource_policies_mock_test.go b/resources/services/organizations/resource_policies_mock_test.go index 6cdac9a3..f95c3f16 100644 --- a/resources/services/organizations/resource_policies_mock_test.go +++ b/resources/services/organizations/resource_policies_mock_test.go @@ -3,12 +3,13 @@ package organizations import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/organizations" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildResourcePolicy(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/organizations/roots.go b/resources/services/organizations/roots.go index b39c5e52..14a6f58f 100644 --- a/resources/services/organizations/roots.go +++ b/resources/services/organizations/roots.go @@ -3,13 +3,14 @@ package organizations import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/organizations" "github.com/aws/aws-sdk-go-v2/service/organizations/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Roots() *schema.Table { @@ -38,6 +39,7 @@ The 'request_account_id' column is added to show from where the request was made }, } } + func fetchOrganizationsRoots(ctx context.Context, meta schema.ClientMeta, _ *schema.Resource, res chan<- any) error { cl := meta.(*client.Client) svc := cl.Services(client.AWSServiceOrganizations).Organizations diff --git a/resources/services/organizations/roots_mock_test.go b/resources/services/organizations/roots_mock_test.go index 098ecba8..29014066 100644 --- a/resources/services/organizations/roots_mock_test.go +++ b/resources/services/organizations/roots_mock_test.go @@ -3,13 +3,14 @@ package organizations import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/organizations" "github.com/aws/aws-sdk-go-v2/service/organizations/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildOrganizationsRoots(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/qldb/ledger_journal_kinesis_streams.go b/resources/services/qldb/ledger_journal_kinesis_streams.go index e76036a7..0df6162d 100644 --- a/resources/services/qldb/ledger_journal_kinesis_streams.go +++ b/resources/services/qldb/ledger_journal_kinesis_streams.go @@ -3,13 +3,14 @@ package qldb import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/qldb" "github.com/aws/aws-sdk-go-v2/service/qldb/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ledgerJournalKinesisStreams() *schema.Table { diff --git a/resources/services/qldb/ledger_journal_s3_exports.go b/resources/services/qldb/ledger_journal_s3_exports.go index c2c8dcc6..b66ac93d 100644 --- a/resources/services/qldb/ledger_journal_s3_exports.go +++ b/resources/services/qldb/ledger_journal_s3_exports.go @@ -3,13 +3,14 @@ package qldb import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/qldb" "github.com/aws/aws-sdk-go-v2/service/qldb/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ledgerJournalS3Exports() *schema.Table { diff --git a/resources/services/qldb/ledgers.go b/resources/services/qldb/ledgers.go index fff70358..ed89ab29 100644 --- a/resources/services/qldb/ledgers.go +++ b/resources/services/qldb/ledgers.go @@ -3,13 +3,14 @@ package qldb import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/qldb" "github.com/aws/aws-sdk-go-v2/service/qldb/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Ledgers() *schema.Table { diff --git a/resources/services/qldb/ledgers_mock_test.go b/resources/services/qldb/ledgers_mock_test.go index 0824cf16..548f86ea 100644 --- a/resources/services/qldb/ledgers_mock_test.go +++ b/resources/services/qldb/ledgers_mock_test.go @@ -3,13 +3,14 @@ package qldb import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/qldb" "github.com/aws/aws-sdk-go-v2/service/qldb/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildLedgersMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/quicksight/analyses.go b/resources/services/quicksight/analyses.go index 13ab6c16..5696a636 100644 --- a/resources/services/quicksight/analyses.go +++ b/resources/services/quicksight/analyses.go @@ -3,15 +3,16 @@ package quicksight import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/quicksight" "github.com/aws/aws-sdk-go-v2/service/quicksight/types" "github.com/aws/smithy-go" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" "github.com/pkg/errors" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Analyses() *schema.Table { diff --git a/resources/services/quicksight/analyses_mock_test.go b/resources/services/quicksight/analyses_mock_test.go index c0b79298..4f367de1 100644 --- a/resources/services/quicksight/analyses_mock_test.go +++ b/resources/services/quicksight/analyses_mock_test.go @@ -3,12 +3,13 @@ package quicksight import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/quicksight" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildAnalysesMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -34,6 +35,7 @@ func buildAnalysesMock(t *testing.T, ctrl *gomock.Controller) client.Services { Quicksight: m, } } + func TestQuicksightAnalyses(t *testing.T) { client.AwsMockTestHelper(t, Analyses(), buildAnalysesMock, client.TestOptions{}) } diff --git a/resources/services/quicksight/dashboards.go b/resources/services/quicksight/dashboards.go index 87f0eba5..eb37ecd5 100644 --- a/resources/services/quicksight/dashboards.go +++ b/resources/services/quicksight/dashboards.go @@ -3,15 +3,16 @@ package quicksight import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/quicksight" "github.com/aws/aws-sdk-go-v2/service/quicksight/types" "github.com/aws/smithy-go" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" "github.com/pkg/errors" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Dashboards() *schema.Table { diff --git a/resources/services/quicksight/dashboards_mock_test.go b/resources/services/quicksight/dashboards_mock_test.go index ace101ad..8540cb26 100644 --- a/resources/services/quicksight/dashboards_mock_test.go +++ b/resources/services/quicksight/dashboards_mock_test.go @@ -3,12 +3,13 @@ package quicksight import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/quicksight" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDashboardsMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -29,6 +30,7 @@ func buildDashboardsMock(t *testing.T, ctrl *gomock.Controller) client.Services Quicksight: m, } } + func TestQuicksightDashboards(t *testing.T) { client.AwsMockTestHelper(t, Dashboards(), buildDashboardsMock, client.TestOptions{}) } diff --git a/resources/services/quicksight/data_sets.go b/resources/services/quicksight/data_sets.go index 87cd4ef6..ded3fa17 100644 --- a/resources/services/quicksight/data_sets.go +++ b/resources/services/quicksight/data_sets.go @@ -3,15 +3,16 @@ package quicksight import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/quicksight" "github.com/aws/aws-sdk-go-v2/service/quicksight/types" "github.com/aws/smithy-go" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" "github.com/pkg/errors" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func DataSets() *schema.Table { diff --git a/resources/services/quicksight/data_sets_mock_test.go b/resources/services/quicksight/data_sets_mock_test.go index f85f63e4..efe5824d 100644 --- a/resources/services/quicksight/data_sets_mock_test.go +++ b/resources/services/quicksight/data_sets_mock_test.go @@ -3,12 +3,13 @@ package quicksight import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/quicksight" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDataSetsMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -40,6 +41,7 @@ func buildDataSetsMock(t *testing.T, ctrl *gomock.Controller) client.Services { Quicksight: m, } } + func TestQuicksightDataSets(t *testing.T) { client.AwsMockTestHelper(t, DataSets(), buildDataSetsMock, client.TestOptions{}) } diff --git a/resources/services/quicksight/data_sources.go b/resources/services/quicksight/data_sources.go index 6dd3549e..d37b0862 100644 --- a/resources/services/quicksight/data_sources.go +++ b/resources/services/quicksight/data_sources.go @@ -3,15 +3,16 @@ package quicksight import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/quicksight" "github.com/aws/aws-sdk-go-v2/service/quicksight/types" "github.com/aws/smithy-go" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" "github.com/pkg/errors" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func DataSources() *schema.Table { diff --git a/resources/services/quicksight/data_sources_mock_test.go b/resources/services/quicksight/data_sources_mock_test.go index 433d7a87..b4caa7b9 100644 --- a/resources/services/quicksight/data_sources_mock_test.go +++ b/resources/services/quicksight/data_sources_mock_test.go @@ -3,12 +3,13 @@ package quicksight import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/quicksight" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDataSourcesMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -29,6 +30,7 @@ func buildDataSourcesMock(t *testing.T, ctrl *gomock.Controller) client.Services Quicksight: m, } } + func TestQuicksightDataSources(t *testing.T) { client.AwsMockTestHelper(t, DataSources(), buildDataSourcesMock, client.TestOptions{}) } diff --git a/resources/services/quicksight/folders.go b/resources/services/quicksight/folders.go index b3c12c9f..a09ce8b2 100644 --- a/resources/services/quicksight/folders.go +++ b/resources/services/quicksight/folders.go @@ -3,15 +3,16 @@ package quicksight import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/quicksight" "github.com/aws/aws-sdk-go-v2/service/quicksight/types" "github.com/aws/smithy-go" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" "github.com/pkg/errors" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Folders() *schema.Table { diff --git a/resources/services/quicksight/folders_mock_test.go b/resources/services/quicksight/folders_mock_test.go index a5674e2e..7be81260 100644 --- a/resources/services/quicksight/folders_mock_test.go +++ b/resources/services/quicksight/folders_mock_test.go @@ -3,12 +3,13 @@ package quicksight import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/quicksight" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildFoldersMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -34,6 +35,7 @@ func buildFoldersMock(t *testing.T, ctrl *gomock.Controller) client.Services { Quicksight: m, } } + func TestQuicksightFolders(t *testing.T) { client.AwsMockTestHelper(t, Folders(), buildFoldersMock, client.TestOptions{}) } diff --git a/resources/services/quicksight/group_members.go b/resources/services/quicksight/group_members.go index 54fec55c..dbe6f591 100644 --- a/resources/services/quicksight/group_members.go +++ b/resources/services/quicksight/group_members.go @@ -3,13 +3,14 @@ package quicksight import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/quicksight" "github.com/aws/aws-sdk-go-v2/service/quicksight/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func groupMembers() *schema.Table { diff --git a/resources/services/quicksight/groups.go b/resources/services/quicksight/groups.go index 8c0f6ef0..d5cddbd1 100644 --- a/resources/services/quicksight/groups.go +++ b/resources/services/quicksight/groups.go @@ -3,15 +3,16 @@ package quicksight import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/quicksight" "github.com/aws/aws-sdk-go-v2/service/quicksight/types" "github.com/aws/smithy-go" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" "github.com/pkg/errors" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Groups() *schema.Table { diff --git a/resources/services/quicksight/groups_mock_test.go b/resources/services/quicksight/groups_mock_test.go index 6e7b72dd..aa8695ce 100644 --- a/resources/services/quicksight/groups_mock_test.go +++ b/resources/services/quicksight/groups_mock_test.go @@ -3,12 +3,13 @@ package quicksight import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/quicksight" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildGroupsMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -35,6 +36,7 @@ func buildGroupsMock(t *testing.T, ctrl *gomock.Controller) client.Services { Quicksight: m, } } + func TestQuicksightGroups(t *testing.T) { client.AwsMockTestHelper(t, Groups(), buildGroupsMock, client.TestOptions{}) } diff --git a/resources/services/quicksight/ingestions.go b/resources/services/quicksight/ingestions.go index 338ffdc2..257669d1 100644 --- a/resources/services/quicksight/ingestions.go +++ b/resources/services/quicksight/ingestions.go @@ -3,13 +3,14 @@ package quicksight import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/quicksight" "github.com/aws/aws-sdk-go-v2/service/quicksight/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ingestions() *schema.Table { diff --git a/resources/services/quicksight/tag_fetch.go b/resources/services/quicksight/tag_fetch.go index 2d2af5da..60571215 100644 --- a/resources/services/quicksight/tag_fetch.go +++ b/resources/services/quicksight/tag_fetch.go @@ -3,11 +3,12 @@ package quicksight import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/service/quicksight" - "github.com/cloudquery/plugin-sdk/v4/schema" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" "github.com/thoas/go-funk" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) var tagsCol = schema.Column{ diff --git a/resources/services/quicksight/templates.go b/resources/services/quicksight/templates.go index f7c19e7a..2aaf833f 100644 --- a/resources/services/quicksight/templates.go +++ b/resources/services/quicksight/templates.go @@ -3,15 +3,16 @@ package quicksight import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/quicksight" "github.com/aws/aws-sdk-go-v2/service/quicksight/types" "github.com/aws/smithy-go" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" "github.com/pkg/errors" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Templates() *schema.Table { diff --git a/resources/services/quicksight/templates_mock_test.go b/resources/services/quicksight/templates_mock_test.go index 6a68bee9..6c2b28ac 100644 --- a/resources/services/quicksight/templates_mock_test.go +++ b/resources/services/quicksight/templates_mock_test.go @@ -3,12 +3,13 @@ package quicksight import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/quicksight" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildTemplatesMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -29,6 +30,7 @@ func buildTemplatesMock(t *testing.T, ctrl *gomock.Controller) client.Services { Quicksight: m, } } + func TestQuicksightTemplates(t *testing.T) { client.AwsMockTestHelper(t, Templates(), buildTemplatesMock, client.TestOptions{}) } diff --git a/resources/services/quicksight/users.go b/resources/services/quicksight/users.go index b3a8a07b..725b13ba 100644 --- a/resources/services/quicksight/users.go +++ b/resources/services/quicksight/users.go @@ -3,15 +3,16 @@ package quicksight import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/quicksight" "github.com/aws/aws-sdk-go-v2/service/quicksight/types" "github.com/aws/smithy-go" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" "github.com/pkg/errors" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Users() *schema.Table { diff --git a/resources/services/quicksight/users_mock_test.go b/resources/services/quicksight/users_mock_test.go index 5f72f621..5622c692 100644 --- a/resources/services/quicksight/users_mock_test.go +++ b/resources/services/quicksight/users_mock_test.go @@ -3,12 +3,13 @@ package quicksight import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/quicksight" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildUsersMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -29,6 +30,7 @@ func buildUsersMock(t *testing.T, ctrl *gomock.Controller) client.Services { Quicksight: m, } } + func TestQuicksightUsers(t *testing.T) { client.AwsMockTestHelper(t, Users(), buildUsersMock, client.TestOptions{}) } diff --git a/resources/services/ram/resource_share_invitations.go b/resources/services/ram/resource_share_invitations.go index 6599a373..fa6bea27 100644 --- a/resources/services/ram/resource_share_invitations.go +++ b/resources/services/ram/resource_share_invitations.go @@ -4,12 +4,13 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/ram" "github.com/aws/aws-sdk-go-v2/service/ram/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ResourceShareInvitations() *schema.Table { diff --git a/resources/services/ram/resource_share_invitations_mock_test.go b/resources/services/ram/resource_share_invitations_mock_test.go index 379282ee..36607d56 100644 --- a/resources/services/ram/resource_share_invitations_mock_test.go +++ b/resources/services/ram/resource_share_invitations_mock_test.go @@ -3,13 +3,14 @@ package ram import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ram" "github.com/aws/aws-sdk-go-v2/service/ram/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRamResourceShareInvitationsMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -26,6 +27,7 @@ func buildRamResourceShareInvitationsMock(t *testing.T, ctrl *gomock.Controller) Ram: m, } } + func TestRamResourceShareInvitations(t *testing.T) { client.AwsMockTestHelper(t, ResourceShareInvitations(), buildRamResourceShareInvitationsMock, client.TestOptions{}) } diff --git a/resources/services/ram/resource_share_permissions.go b/resources/services/ram/resource_share_permissions.go index 9a3889f0..0cb04082 100644 --- a/resources/services/ram/resource_share_permissions.go +++ b/resources/services/ram/resource_share_permissions.go @@ -4,14 +4,15 @@ import ( "context" "strconv" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ram" "github.com/aws/aws-sdk-go-v2/service/ram/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func resourceSharePermissions() *schema.Table { diff --git a/resources/services/ram/resource_shares.go b/resources/services/ram/resource_shares.go index b100ab24..6b7c8ad3 100644 --- a/resources/services/ram/resource_shares.go +++ b/resources/services/ram/resource_shares.go @@ -3,14 +3,15 @@ package ram import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ram" "github.com/aws/aws-sdk-go-v2/service/ram/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func ResourceShares() *schema.Table { diff --git a/resources/services/ram/resource_shares_mock_test.go b/resources/services/ram/resource_shares_mock_test.go index e38cf605..9cebd2ea 100644 --- a/resources/services/ram/resource_shares_mock_test.go +++ b/resources/services/ram/resource_shares_mock_test.go @@ -4,13 +4,14 @@ import ( "fmt" "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ram" "github.com/aws/aws-sdk-go-v2/service/ram/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRamResourceSharesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ram/resources.go b/resources/services/ram/resources.go index 6048f152..b16676e3 100644 --- a/resources/services/ram/resources.go +++ b/resources/services/ram/resources.go @@ -3,13 +3,14 @@ package ram import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ram" "github.com/aws/aws-sdk-go-v2/service/ram/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Resources() *schema.Table { diff --git a/resources/services/ram/resources_mock_test.go b/resources/services/ram/resources_mock_test.go index c6b0cffa..b8694197 100644 --- a/resources/services/ram/resources_mock_test.go +++ b/resources/services/ram/resources_mock_test.go @@ -3,13 +3,14 @@ package ram import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ram" "github.com/aws/aws-sdk-go-v2/service/ram/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRamResourcesMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -26,6 +27,7 @@ func buildRamResourcesMock(t *testing.T, ctrl *gomock.Controller) client.Service Ram: m, } } + func TestRamResources(t *testing.T) { client.AwsMockTestHelper(t, Resources(), buildRamResourcesMock, client.TestOptions{}) } diff --git a/resources/services/rds/certificates.go b/resources/services/rds/certificates.go index c44b9936..a7293f2c 100644 --- a/resources/services/rds/certificates.go +++ b/resources/services/rds/certificates.go @@ -3,12 +3,13 @@ package rds import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Certificates() *schema.Table { diff --git a/resources/services/rds/cluster_backtracks.go b/resources/services/rds/cluster_backtracks.go index 60ecac57..fe5ef7c2 100644 --- a/resources/services/rds/cluster_backtracks.go +++ b/resources/services/rds/cluster_backtracks.go @@ -3,13 +3,14 @@ package rds import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func clusterBacktracks() *schema.Table { diff --git a/resources/services/rds/cluster_backtracks_mock_test.go b/resources/services/rds/cluster_backtracks_mock_test.go index 11076139..990cf323 100644 --- a/resources/services/rds/cluster_backtracks_mock_test.go +++ b/resources/services/rds/cluster_backtracks_mock_test.go @@ -3,12 +3,13 @@ package rds import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRdsClusterBacktracks(t *testing.T, mockRds *mocks.MockRdsClient) { diff --git a/resources/services/rds/cluster_parameter_group_parameters.go b/resources/services/rds/cluster_parameter_group_parameters.go index 52ad808e..add0e37e 100644 --- a/resources/services/rds/cluster_parameter_group_parameters.go +++ b/resources/services/rds/cluster_parameter_group_parameters.go @@ -3,12 +3,13 @@ package rds import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func clusterParameterGroupParameters() *schema.Table { diff --git a/resources/services/rds/cluster_parameter_groups.go b/resources/services/rds/cluster_parameter_groups.go index 9fe8d942..fd0a856d 100644 --- a/resources/services/rds/cluster_parameter_groups.go +++ b/resources/services/rds/cluster_parameter_groups.go @@ -3,13 +3,14 @@ package rds import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func ClusterParameterGroups() *schema.Table { diff --git a/resources/services/rds/cluster_parameter_groups_mock_test.go b/resources/services/rds/cluster_parameter_groups_mock_test.go index 124b1c51..2aa89d0d 100644 --- a/resources/services/rds/cluster_parameter_groups_mock_test.go +++ b/resources/services/rds/cluster_parameter_groups_mock_test.go @@ -3,14 +3,15 @@ package rds import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRdsClusterParameterGroups(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/rds/cluster_snapshots.go b/resources/services/rds/cluster_snapshots.go index 5b5f1f97..43b01656 100644 --- a/resources/services/rds/cluster_snapshots.go +++ b/resources/services/rds/cluster_snapshots.go @@ -3,13 +3,14 @@ package rds import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func ClusterSnapshots() *schema.Table { diff --git a/resources/services/rds/cluster_snapshots_mock_test.go b/resources/services/rds/cluster_snapshots_mock_test.go index 8b7941ca..12d906de 100644 --- a/resources/services/rds/cluster_snapshots_mock_test.go +++ b/resources/services/rds/cluster_snapshots_mock_test.go @@ -3,13 +3,14 @@ package rds import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRDSClientForClusterSnapshots(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/rds/clusters.go b/resources/services/rds/clusters.go index 07ea9a01..aec757a1 100644 --- a/resources/services/rds/clusters.go +++ b/resources/services/rds/clusters.go @@ -3,13 +3,14 @@ package rds import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Clusters() *schema.Table { diff --git a/resources/services/rds/db_parameter_group_db_parameters.go b/resources/services/rds/db_parameter_group_db_parameters.go index f588fa31..44e54813 100644 --- a/resources/services/rds/db_parameter_group_db_parameters.go +++ b/resources/services/rds/db_parameter_group_db_parameters.go @@ -1,11 +1,12 @@ package rds import ( - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/rds/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func dbParameterGroupDbParameters() *schema.Table { diff --git a/resources/services/rds/db_parameter_groups.go b/resources/services/rds/db_parameter_groups.go index 26c0cf8b..49baf843 100644 --- a/resources/services/rds/db_parameter_groups.go +++ b/resources/services/rds/db_parameter_groups.go @@ -3,13 +3,14 @@ package rds import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func DbParameterGroups() *schema.Table { diff --git a/resources/services/rds/db_parameter_groups_mock_test.go b/resources/services/rds/db_parameter_groups_mock_test.go index e31405bc..ca07fb63 100644 --- a/resources/services/rds/db_parameter_groups_mock_test.go +++ b/resources/services/rds/db_parameter_groups_mock_test.go @@ -3,14 +3,15 @@ package rds import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRDSDBParameterGroups(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/rds/db_proxies.go b/resources/services/rds/db_proxies.go index a0971694..ce8d2f69 100644 --- a/resources/services/rds/db_proxies.go +++ b/resources/services/rds/db_proxies.go @@ -3,13 +3,14 @@ package rds import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func DbProxies() *schema.Table { diff --git a/resources/services/rds/db_proxies_mock_test.go b/resources/services/rds/db_proxies_mock_test.go index 46463ddc..28533068 100644 --- a/resources/services/rds/db_proxies_mock_test.go +++ b/resources/services/rds/db_proxies_mock_test.go @@ -3,13 +3,14 @@ package rds import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRdsDbProxiesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/rds/db_security_groups.go b/resources/services/rds/db_security_groups.go index 115fb6e0..a68ee6d0 100644 --- a/resources/services/rds/db_security_groups.go +++ b/resources/services/rds/db_security_groups.go @@ -3,13 +3,14 @@ package rds import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func DbSecurityGroups() *schema.Table { diff --git a/resources/services/rds/db_security_groups_mock_test.go b/resources/services/rds/db_security_groups_mock_test.go index def8bdb5..978d7d20 100644 --- a/resources/services/rds/db_security_groups_mock_test.go +++ b/resources/services/rds/db_security_groups_mock_test.go @@ -3,14 +3,15 @@ package rds import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRdsDbSecurityGroups(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/rds/db_snapshots.go b/resources/services/rds/db_snapshots.go index 4088ea1e..0212f2d4 100644 --- a/resources/services/rds/db_snapshots.go +++ b/resources/services/rds/db_snapshots.go @@ -3,14 +3,15 @@ package rds import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/rds/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/rds/models" ) func DbSnapshots() *schema.Table { diff --git a/resources/services/rds/db_snapshots_mock_test.go b/resources/services/rds/db_snapshots_mock_test.go index 8c558939..21c2c309 100644 --- a/resources/services/rds/db_snapshots_mock_test.go +++ b/resources/services/rds/db_snapshots_mock_test.go @@ -3,13 +3,14 @@ package rds import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRDSClient(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/rds/event_subscriptions.go b/resources/services/rds/event_subscriptions.go index 229cbaaa..4e4ad6bd 100644 --- a/resources/services/rds/event_subscriptions.go +++ b/resources/services/rds/event_subscriptions.go @@ -3,13 +3,14 @@ package rds import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func EventSubscriptions() *schema.Table { diff --git a/resources/services/rds/event_subscriptions_mock_test.go b/resources/services/rds/event_subscriptions_mock_test.go index 1caf6bc1..6805b521 100644 --- a/resources/services/rds/event_subscriptions_mock_test.go +++ b/resources/services/rds/event_subscriptions_mock_test.go @@ -3,14 +3,15 @@ package rds import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRDSEventSubscriptions(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/rds/fetch_rds_tags.go b/resources/services/rds/fetch_rds_tags.go index 406146ef..6842559d 100644 --- a/resources/services/rds/fetch_rds_tags.go +++ b/resources/services/rds/fetch_rds_tags.go @@ -3,10 +3,11 @@ package rds import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/service/rds" - "github.com/cloudquery/plugin-sdk/v4/schema" "github.com/thoas/go-funk" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" ) func resolveRDSTags(path string) schema.ColumnResolver { diff --git a/resources/services/rds/instances.go b/resources/services/rds/instances.go index b7f1db1b..baa45a7f 100644 --- a/resources/services/rds/instances.go +++ b/resources/services/rds/instances.go @@ -3,13 +3,14 @@ package rds import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Instances() *schema.Table { diff --git a/resources/services/rds/mock_test.go b/resources/services/rds/mock_test.go index 29667b29..ee5854a7 100644 --- a/resources/services/rds/mock_test.go +++ b/resources/services/rds/mock_test.go @@ -3,14 +3,15 @@ package rds import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/rds" rdsTypes "github.com/aws/aws-sdk-go-v2/service/rds/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRdsCertificates(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -104,12 +105,15 @@ func buildRdsDBReservedInstances(t *testing.T, ctrl *gomock.Controller) client.S func TestRdsCertificates(t *testing.T) { client.AwsMockTestHelper(t, Certificates(), buildRdsCertificates, client.TestOptions{}) } + func TestRdsInstances(t *testing.T) { client.AwsMockTestHelper(t, Instances(), buildRdsDBInstances, client.TestOptions{}) } + func TestRdsClusters(t *testing.T) { client.AwsMockTestHelper(t, Clusters(), buildRdsDBClusters, client.TestOptions{}) } + func TestRdsSubnetGroups(t *testing.T) { client.AwsMockTestHelper(t, SubnetGroups(), buildRdsDBSubnetGroups, client.TestOptions{}) } diff --git a/resources/services/rds/option_groups.go b/resources/services/rds/option_groups.go index 7e8a4391..530bf90f 100644 --- a/resources/services/rds/option_groups.go +++ b/resources/services/rds/option_groups.go @@ -3,13 +3,14 @@ package rds import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func OptionGroups() *schema.Table { diff --git a/resources/services/rds/option_groups_mock_test.go b/resources/services/rds/option_groups_mock_test.go index e19a4edb..34660974 100644 --- a/resources/services/rds/option_groups_mock_test.go +++ b/resources/services/rds/option_groups_mock_test.go @@ -3,14 +3,15 @@ package rds import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildOptionGroups(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/rds/reserved_instances.go b/resources/services/rds/reserved_instances.go index 925043a0..2e672814 100644 --- a/resources/services/rds/reserved_instances.go +++ b/resources/services/rds/reserved_instances.go @@ -3,13 +3,14 @@ package rds import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func ReservedInstances() *schema.Table { diff --git a/resources/services/rds/subnet_groups.go b/resources/services/rds/subnet_groups.go index ae59b52e..992273df 100644 --- a/resources/services/rds/subnet_groups.go +++ b/resources/services/rds/subnet_groups.go @@ -3,13 +3,14 @@ package rds import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func SubnetGroups() *schema.Table { diff --git a/resources/services/redshift/cluster_parameter_groups.go b/resources/services/redshift/cluster_parameter_groups.go index 10743cbe..21271121 100644 --- a/resources/services/redshift/cluster_parameter_groups.go +++ b/resources/services/redshift/cluster_parameter_groups.go @@ -3,11 +3,12 @@ package redshift import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/redshift/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func clusterParameterGroups() *schema.Table { diff --git a/resources/services/redshift/cluster_parameters.go b/resources/services/redshift/cluster_parameters.go index c854e55e..853342b7 100644 --- a/resources/services/redshift/cluster_parameters.go +++ b/resources/services/redshift/cluster_parameters.go @@ -3,12 +3,13 @@ package redshift import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/redshift" "github.com/aws/aws-sdk-go-v2/service/redshift/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func clusterParameters() *schema.Table { diff --git a/resources/services/redshift/clusters.go b/resources/services/redshift/clusters.go index d18c9e1e..159d7966 100644 --- a/resources/services/redshift/clusters.go +++ b/resources/services/redshift/clusters.go @@ -4,13 +4,14 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/redshift" "github.com/aws/aws-sdk-go-v2/service/redshift/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Clusters() *schema.Table { diff --git a/resources/services/redshift/clusters_mock_test.go b/resources/services/redshift/clusters_mock_test.go index a098348b..cebb3e0f 100644 --- a/resources/services/redshift/clusters_mock_test.go +++ b/resources/services/redshift/clusters_mock_test.go @@ -3,14 +3,15 @@ package redshift import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/redshift" "github.com/aws/aws-sdk-go-v2/service/redshift/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildClustersMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/redshift/data_shares.go b/resources/services/redshift/data_shares.go index 264846d3..c95cb8b2 100644 --- a/resources/services/redshift/data_shares.go +++ b/resources/services/redshift/data_shares.go @@ -3,13 +3,14 @@ package redshift import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/redshift" "github.com/aws/aws-sdk-go-v2/service/redshift/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func DataShares() *schema.Table { diff --git a/resources/services/redshift/data_shares_mock_test.go b/resources/services/redshift/data_shares_mock_test.go index cdb6412b..f1ea3155 100644 --- a/resources/services/redshift/data_shares_mock_test.go +++ b/resources/services/redshift/data_shares_mock_test.go @@ -3,13 +3,14 @@ package redshift import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/redshift" "github.com/aws/aws-sdk-go-v2/service/redshift/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDataSharesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/redshift/endpoint_access.go b/resources/services/redshift/endpoint_access.go index d180f4f1..05a40c55 100644 --- a/resources/services/redshift/endpoint_access.go +++ b/resources/services/redshift/endpoint_access.go @@ -3,13 +3,14 @@ package redshift import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/redshift" "github.com/aws/aws-sdk-go-v2/service/redshift/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func endpointAccess() *schema.Table { diff --git a/resources/services/redshift/endpoint_authorization.go b/resources/services/redshift/endpoint_authorization.go index b9820838..aee4d422 100644 --- a/resources/services/redshift/endpoint_authorization.go +++ b/resources/services/redshift/endpoint_authorization.go @@ -3,13 +3,14 @@ package redshift import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/redshift" "github.com/aws/aws-sdk-go-v2/service/redshift/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func endpointAuthorization() *schema.Table { diff --git a/resources/services/redshift/event_subscriptions.go b/resources/services/redshift/event_subscriptions.go index 53aa9ac1..4fdc7747 100644 --- a/resources/services/redshift/event_subscriptions.go +++ b/resources/services/redshift/event_subscriptions.go @@ -4,15 +4,16 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/redshift" "github.com/aws/aws-sdk-go-v2/service/redshift/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func EventSubscriptions() *schema.Table { diff --git a/resources/services/redshift/event_subscriptions_mock_test.go b/resources/services/redshift/event_subscriptions_mock_test.go index 83095606..c06eedcf 100644 --- a/resources/services/redshift/event_subscriptions_mock_test.go +++ b/resources/services/redshift/event_subscriptions_mock_test.go @@ -3,14 +3,15 @@ package redshift import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/redshift" "github.com/aws/aws-sdk-go-v2/service/redshift/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEventSubscriptionsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/redshift/snapshots.go b/resources/services/redshift/snapshots.go index afb7b5bb..4a0333ff 100644 --- a/resources/services/redshift/snapshots.go +++ b/resources/services/redshift/snapshots.go @@ -4,15 +4,16 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/redshift" "github.com/aws/aws-sdk-go-v2/service/redshift/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func snapshots() *schema.Table { diff --git a/resources/services/redshift/subnet_groups.go b/resources/services/redshift/subnet_groups.go index 550f0d56..9f2b12f7 100644 --- a/resources/services/redshift/subnet_groups.go +++ b/resources/services/redshift/subnet_groups.go @@ -4,13 +4,14 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/redshift" "github.com/aws/aws-sdk-go-v2/service/redshift/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func SubnetGroups() *schema.Table { diff --git a/resources/services/redshift/subnet_groups_mock_test.go b/resources/services/redshift/subnet_groups_mock_test.go index bf90b312..3b39fb5d 100644 --- a/resources/services/redshift/subnet_groups_mock_test.go +++ b/resources/services/redshift/subnet_groups_mock_test.go @@ -3,13 +3,14 @@ package redshift import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/redshift" "github.com/aws/aws-sdk-go-v2/service/redshift/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildSubnetGroupsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/resiliencehub/alarm_recommendations.go b/resources/services/resiliencehub/alarm_recommendations.go index 4974a451..5464c866 100644 --- a/resources/services/resiliencehub/alarm_recommendations.go +++ b/resources/services/resiliencehub/alarm_recommendations.go @@ -3,11 +3,12 @@ package resiliencehub import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/service/resiliencehub" "github.com/aws/aws-sdk-go-v2/service/resiliencehub/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func alarmRecommendations() *schema.Table { diff --git a/resources/services/resiliencehub/alarm_recommendations_mock_test.go b/resources/services/resiliencehub/alarm_recommendations_mock_test.go index 16a340c4..11818546 100644 --- a/resources/services/resiliencehub/alarm_recommendations_mock_test.go +++ b/resources/services/resiliencehub/alarm_recommendations_mock_test.go @@ -3,11 +3,12 @@ package resiliencehub import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/resiliencehub" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildAppAlarmRecommendations(t *testing.T, mock *mocks.MockResiliencehubClient) { diff --git a/resources/services/resiliencehub/app_assessments.go b/resources/services/resiliencehub/app_assessments.go index c5492d8f..60fef4cd 100644 --- a/resources/services/resiliencehub/app_assessments.go +++ b/resources/services/resiliencehub/app_assessments.go @@ -3,11 +3,12 @@ package resiliencehub import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/service/resiliencehub" "github.com/aws/aws-sdk-go-v2/service/resiliencehub/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func appAssesments() *schema.Table { diff --git a/resources/services/resiliencehub/app_assessments_mock_test.go b/resources/services/resiliencehub/app_assessments_mock_test.go index a26e278a..44424d93 100644 --- a/resources/services/resiliencehub/app_assessments_mock_test.go +++ b/resources/services/resiliencehub/app_assessments_mock_test.go @@ -3,11 +3,12 @@ package resiliencehub import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/resiliencehub" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildAppAssessments(t *testing.T, mock *mocks.MockResiliencehubClient) { diff --git a/resources/services/resiliencehub/app_component_compliances.go b/resources/services/resiliencehub/app_component_compliances.go index 19ddc2ae..8f7ea70a 100644 --- a/resources/services/resiliencehub/app_component_compliances.go +++ b/resources/services/resiliencehub/app_component_compliances.go @@ -3,11 +3,12 @@ package resiliencehub import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/service/resiliencehub" "github.com/aws/aws-sdk-go-v2/service/resiliencehub/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func appComponentCompliances() *schema.Table { diff --git a/resources/services/resiliencehub/app_component_compliances_mock_test.go b/resources/services/resiliencehub/app_component_compliances_mock_test.go index 4bda11e7..d8f8e6ee 100644 --- a/resources/services/resiliencehub/app_component_compliances_mock_test.go +++ b/resources/services/resiliencehub/app_component_compliances_mock_test.go @@ -3,11 +3,12 @@ package resiliencehub import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/resiliencehub" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildAppComponentCompliances(t *testing.T, mock *mocks.MockResiliencehubClient) { diff --git a/resources/services/resiliencehub/app_version_resource_mappings.go b/resources/services/resiliencehub/app_version_resource_mappings.go index 0ea7a9af..c9e48234 100644 --- a/resources/services/resiliencehub/app_version_resource_mappings.go +++ b/resources/services/resiliencehub/app_version_resource_mappings.go @@ -3,12 +3,13 @@ package resiliencehub import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/resiliencehub" "github.com/aws/aws-sdk-go-v2/service/resiliencehub/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func appVersionResourceMappings() *schema.Table { diff --git a/resources/services/resiliencehub/app_version_resource_mappings_mock_test.go b/resources/services/resiliencehub/app_version_resource_mappings_mock_test.go index 9a71e305..4074e263 100644 --- a/resources/services/resiliencehub/app_version_resource_mappings_mock_test.go +++ b/resources/services/resiliencehub/app_version_resource_mappings_mock_test.go @@ -3,11 +3,12 @@ package resiliencehub import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/resiliencehub" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildAppVersionResourceMappings(t *testing.T, mock *mocks.MockResiliencehubClient) { diff --git a/resources/services/resiliencehub/app_version_resources.go b/resources/services/resiliencehub/app_version_resources.go index 4bbb2a09..824930af 100644 --- a/resources/services/resiliencehub/app_version_resources.go +++ b/resources/services/resiliencehub/app_version_resources.go @@ -3,13 +3,14 @@ package resiliencehub import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/resiliencehub" "github.com/aws/aws-sdk-go-v2/service/resiliencehub/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func appVersionResources() *schema.Table { diff --git a/resources/services/resiliencehub/app_version_resources_mock_test.go b/resources/services/resiliencehub/app_version_resources_mock_test.go index 2efaf93e..8114eda1 100644 --- a/resources/services/resiliencehub/app_version_resources_mock_test.go +++ b/resources/services/resiliencehub/app_version_resources_mock_test.go @@ -3,11 +3,12 @@ package resiliencehub import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/resiliencehub" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildAppVersionResources(t *testing.T, mock *mocks.MockResiliencehubClient) { diff --git a/resources/services/resiliencehub/app_versions.go b/resources/services/resiliencehub/app_versions.go index c3ebc965..0b5a9332 100644 --- a/resources/services/resiliencehub/app_versions.go +++ b/resources/services/resiliencehub/app_versions.go @@ -3,11 +3,12 @@ package resiliencehub import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/service/resiliencehub" "github.com/aws/aws-sdk-go-v2/service/resiliencehub/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func appVersions() *schema.Table { diff --git a/resources/services/resiliencehub/app_versions_mock_test.go b/resources/services/resiliencehub/app_versions_mock_test.go index 7492d99f..1f1d135c 100644 --- a/resources/services/resiliencehub/app_versions_mock_test.go +++ b/resources/services/resiliencehub/app_versions_mock_test.go @@ -3,11 +3,12 @@ package resiliencehub import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/resiliencehub" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildAppVersions(t *testing.T, mock *mocks.MockResiliencehubClient) { diff --git a/resources/services/resiliencehub/apps.go b/resources/services/resiliencehub/apps.go index 20647a92..f2d1c93b 100644 --- a/resources/services/resiliencehub/apps.go +++ b/resources/services/resiliencehub/apps.go @@ -3,12 +3,13 @@ package resiliencehub import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/resiliencehub" "github.com/aws/aws-sdk-go-v2/service/resiliencehub/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Apps() *schema.Table { diff --git a/resources/services/resiliencehub/apps_mock_test.go b/resources/services/resiliencehub/apps_mock_test.go index 5edcb1ad..ea884588 100644 --- a/resources/services/resiliencehub/apps_mock_test.go +++ b/resources/services/resiliencehub/apps_mock_test.go @@ -3,12 +3,13 @@ package resiliencehub import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/resiliencehub" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildApps(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/resiliencehub/columns.go b/resources/services/resiliencehub/columns.go index bb400080..84413918 100644 --- a/resources/services/resiliencehub/columns.go +++ b/resources/services/resiliencehub/columns.go @@ -2,7 +2,8 @@ package resiliencehub import ( "github.com/apache/arrow/go/v16/arrow" - "github.com/cloudquery/plugin-sdk/v4/schema" + + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" ) var ( diff --git a/resources/services/resiliencehub/component_recommendations.go b/resources/services/resiliencehub/component_recommendations.go index 260008a9..60102b2d 100644 --- a/resources/services/resiliencehub/component_recommendations.go +++ b/resources/services/resiliencehub/component_recommendations.go @@ -3,11 +3,12 @@ package resiliencehub import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/service/resiliencehub" "github.com/aws/aws-sdk-go-v2/service/resiliencehub/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func appComponentRecommendations() *schema.Table { diff --git a/resources/services/resiliencehub/component_recommendations_mock_test.go b/resources/services/resiliencehub/component_recommendations_mock_test.go index 0cdd04d1..db44fd48 100644 --- a/resources/services/resiliencehub/component_recommendations_mock_test.go +++ b/resources/services/resiliencehub/component_recommendations_mock_test.go @@ -3,11 +3,12 @@ package resiliencehub import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/resiliencehub" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildComponentRecommendations(t *testing.T, mock *mocks.MockResiliencehubClient) { diff --git a/resources/services/resiliencehub/recommendation_templates.go b/resources/services/resiliencehub/recommendation_templates.go index 44b2dbfc..6f31648a 100644 --- a/resources/services/resiliencehub/recommendation_templates.go +++ b/resources/services/resiliencehub/recommendation_templates.go @@ -3,11 +3,12 @@ package resiliencehub import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/service/resiliencehub" "github.com/aws/aws-sdk-go-v2/service/resiliencehub/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func recommendationTemplates() *schema.Table { diff --git a/resources/services/resiliencehub/recommendations_templates_mock_test.go b/resources/services/resiliencehub/recommendations_templates_mock_test.go index 8ff26b7d..a0282b30 100644 --- a/resources/services/resiliencehub/recommendations_templates_mock_test.go +++ b/resources/services/resiliencehub/recommendations_templates_mock_test.go @@ -3,11 +3,12 @@ package resiliencehub import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/resiliencehub" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRecommendationsTemplates(t *testing.T, mock *mocks.MockResiliencehubClient) { diff --git a/resources/services/resiliencehub/resiliency_policies.go b/resources/services/resiliencehub/resiliency_policies.go index a4c41d05..2bd6dca1 100644 --- a/resources/services/resiliencehub/resiliency_policies.go +++ b/resources/services/resiliencehub/resiliency_policies.go @@ -3,12 +3,13 @@ package resiliencehub import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/resiliencehub" "github.com/aws/aws-sdk-go-v2/service/resiliencehub/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ResiliencyPolicies() *schema.Table { diff --git a/resources/services/resiliencehub/resiliency_policies_mock_test.go b/resources/services/resiliencehub/resiliency_policies_mock_test.go index 347e4af1..34864466 100644 --- a/resources/services/resiliencehub/resiliency_policies_mock_test.go +++ b/resources/services/resiliencehub/resiliency_policies_mock_test.go @@ -3,12 +3,13 @@ package resiliencehub import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/resiliencehub" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildResiliencyPolicies(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/resiliencehub/sop_recommendations.go b/resources/services/resiliencehub/sop_recommendations.go index 2217cd0b..806c0b80 100644 --- a/resources/services/resiliencehub/sop_recommendations.go +++ b/resources/services/resiliencehub/sop_recommendations.go @@ -3,11 +3,12 @@ package resiliencehub import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/service/resiliencehub" "github.com/aws/aws-sdk-go-v2/service/resiliencehub/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func sopRecommendations() *schema.Table { diff --git a/resources/services/resiliencehub/sop_recommendations_mock_test.go b/resources/services/resiliencehub/sop_recommendations_mock_test.go index e7683f06..b6a0318c 100644 --- a/resources/services/resiliencehub/sop_recommendations_mock_test.go +++ b/resources/services/resiliencehub/sop_recommendations_mock_test.go @@ -3,11 +3,12 @@ package resiliencehub import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/resiliencehub" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildSopAlarmRecommendations(t *testing.T, mock *mocks.MockResiliencehubClient) { diff --git a/resources/services/resiliencehub/suggested_resiliency_policies.go b/resources/services/resiliencehub/suggested_resiliency_policies.go index bd127d78..8931913d 100644 --- a/resources/services/resiliencehub/suggested_resiliency_policies.go +++ b/resources/services/resiliencehub/suggested_resiliency_policies.go @@ -3,12 +3,13 @@ package resiliencehub import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/resiliencehub" "github.com/aws/aws-sdk-go-v2/service/resiliencehub/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func SuggestedResiliencyPolicies() *schema.Table { diff --git a/resources/services/resiliencehub/suggested_resiliency_policies_mock_test.go b/resources/services/resiliencehub/suggested_resiliency_policies_mock_test.go index 110d1fc5..b8eca045 100644 --- a/resources/services/resiliencehub/suggested_resiliency_policies_mock_test.go +++ b/resources/services/resiliencehub/suggested_resiliency_policies_mock_test.go @@ -3,12 +3,13 @@ package resiliencehub import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/resiliencehub" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildSuggestedResiliencyPolicies(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/resiliencehub/test_recommendations.go b/resources/services/resiliencehub/test_recommendations.go index 3d36ce47..5c0782fe 100644 --- a/resources/services/resiliencehub/test_recommendations.go +++ b/resources/services/resiliencehub/test_recommendations.go @@ -3,11 +3,12 @@ package resiliencehub import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/service/resiliencehub" "github.com/aws/aws-sdk-go-v2/service/resiliencehub/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func testRecommendations() *schema.Table { diff --git a/resources/services/resiliencehub/test_recommendations_mock_test.go b/resources/services/resiliencehub/test_recommendations_mock_test.go index 5e3c1362..d86a6407 100644 --- a/resources/services/resiliencehub/test_recommendations_mock_test.go +++ b/resources/services/resiliencehub/test_recommendations_mock_test.go @@ -3,11 +3,12 @@ package resiliencehub import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/resiliencehub" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildAppTestRecommendations(t *testing.T, mock *mocks.MockResiliencehubClient) { diff --git a/resources/services/resourcegroups/resource_groups.go b/resources/services/resourcegroups/resource_groups.go index 160904b5..cc519ca5 100644 --- a/resources/services/resourcegroups/resource_groups.go +++ b/resources/services/resourcegroups/resource_groups.go @@ -3,14 +3,15 @@ package resourcegroups import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/resourcegroups/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/resourcegroups" "github.com/aws/aws-sdk-go-v2/service/resourcegroups/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/resourcegroups/models" ) func ResourceGroups() *schema.Table { @@ -40,6 +41,7 @@ func ResourceGroups() *schema.Table { }, } } + func fetchResourcegroupsResourceGroups(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { var config resourcegroups.ListGroupsInput cl := meta.(*client.Client) diff --git a/resources/services/resourcegroups/resource_groups_mock_test.go b/resources/services/resourcegroups/resource_groups_mock_test.go index b1a44293..fb1560e4 100644 --- a/resources/services/resourcegroups/resource_groups_mock_test.go +++ b/resources/services/resourcegroups/resource_groups_mock_test.go @@ -3,13 +3,14 @@ package resourcegroups import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/resourcegroups" "github.com/aws/aws-sdk-go-v2/service/resourcegroups/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildResourceGroupsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/route53/delegation_sets.go b/resources/services/route53/delegation_sets.go index c38410f2..795351fd 100644 --- a/resources/services/route53/delegation_sets.go +++ b/resources/services/route53/delegation_sets.go @@ -3,13 +3,14 @@ package route53 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/route53" "github.com/aws/aws-sdk-go-v2/service/route53/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func DelegationSets() *schema.Table { diff --git a/resources/services/route53/delegations_sets_mock_test.go b/resources/services/route53/delegations_sets_mock_test.go index a407002b..45c69aac 100644 --- a/resources/services/route53/delegations_sets_mock_test.go +++ b/resources/services/route53/delegations_sets_mock_test.go @@ -3,13 +3,14 @@ package route53 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/route53" route53Types "github.com/aws/aws-sdk-go-v2/service/route53/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRoute53DelegationSetsMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -25,6 +26,7 @@ func buildRoute53DelegationSetsMock(t *testing.T, ctrl *gomock.Controller) clien Route53: m, } } + func TestRoute53DelegationSets(t *testing.T) { client.AwsMockTestHelper(t, DelegationSets(), buildRoute53DelegationSetsMock, client.TestOptions{}) } diff --git a/resources/services/route53/domains.go b/resources/services/route53/domains.go index af0b7da5..cf073a0e 100644 --- a/resources/services/route53/domains.go +++ b/resources/services/route53/domains.go @@ -4,15 +4,16 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/route53domains" "github.com/aws/aws-sdk-go-v2/service/route53domains/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Domains() *schema.Table { @@ -68,6 +69,7 @@ func fetchRoute53Domains(ctx context.Context, meta schema.ClientMeta, parent *sc } return nil } + func getDomain(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource) error { cl := meta.(*client.Client) svc := cl.Services(client.AWSServiceRoute53domains).Route53domains diff --git a/resources/services/route53/domains_mock_test.go b/resources/services/route53/domains_mock_test.go index d6c59e3b..cef31b7f 100644 --- a/resources/services/route53/domains_mock_test.go +++ b/resources/services/route53/domains_mock_test.go @@ -3,13 +3,14 @@ package route53 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/route53domains" "github.com/aws/aws-sdk-go-v2/service/route53domains/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRoute53Domains(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/route53/health_check_mock_test.go b/resources/services/route53/health_check_mock_test.go index 7203e934..3e17fb96 100644 --- a/resources/services/route53/health_check_mock_test.go +++ b/resources/services/route53/health_check_mock_test.go @@ -3,13 +3,14 @@ package route53 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/route53" route53Types "github.com/aws/aws-sdk-go-v2/service/route53/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRoute53HealthChecksMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/route53/health_checks.go b/resources/services/route53/health_checks.go index d8d0329e..9d46dc33 100644 --- a/resources/services/route53/health_checks.go +++ b/resources/services/route53/health_checks.go @@ -3,13 +3,14 @@ package route53 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/route53" "github.com/aws/aws-sdk-go-v2/service/route53/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func HealthChecks() *schema.Table { diff --git a/resources/services/route53/hosted_zone_query_logging_configs.go b/resources/services/route53/hosted_zone_query_logging_configs.go index 5e53414b..3e0d010d 100644 --- a/resources/services/route53/hosted_zone_query_logging_configs.go +++ b/resources/services/route53/hosted_zone_query_logging_configs.go @@ -4,15 +4,16 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/route53/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/route53" "github.com/aws/aws-sdk-go-v2/service/route53/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/route53/models" ) func hostedZoneQueryLoggingConfigs() *schema.Table { diff --git a/resources/services/route53/hosted_zone_resource_record_sets.go b/resources/services/route53/hosted_zone_resource_record_sets.go index aaf81ecc..97c797e7 100644 --- a/resources/services/route53/hosted_zone_resource_record_sets.go +++ b/resources/services/route53/hosted_zone_resource_record_sets.go @@ -3,13 +3,14 @@ package route53 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/route53/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/route53" "github.com/aws/aws-sdk-go-v2/service/route53/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/route53/models" ) func hostedZoneResourceRecordSets() *schema.Table { diff --git a/resources/services/route53/hosted_zone_traffic_policy_instances.go b/resources/services/route53/hosted_zone_traffic_policy_instances.go index d7262b58..21e27192 100644 --- a/resources/services/route53/hosted_zone_traffic_policy_instances.go +++ b/resources/services/route53/hosted_zone_traffic_policy_instances.go @@ -4,15 +4,16 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/route53/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/route53" "github.com/aws/aws-sdk-go-v2/service/route53/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/route53/models" ) func hostedZoneTrafficPolicyInstances() *schema.Table { diff --git a/resources/services/route53/hosted_zones.go b/resources/services/route53/hosted_zones.go index aefd9ba8..91db6204 100644 --- a/resources/services/route53/hosted_zones.go +++ b/resources/services/route53/hosted_zones.go @@ -5,15 +5,16 @@ import ( "fmt" "strings" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/route53/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/route53" "github.com/aws/aws-sdk-go-v2/service/route53/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/route53/models" ) func HostedZones() *schema.Table { diff --git a/resources/services/route53/hosted_zones_mock_test.go b/resources/services/route53/hosted_zones_mock_test.go index 29312a28..a726cf25 100644 --- a/resources/services/route53/hosted_zones_mock_test.go +++ b/resources/services/route53/hosted_zones_mock_test.go @@ -4,13 +4,14 @@ import ( "fmt" "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/route53" "github.com/aws/aws-sdk-go-v2/service/route53/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRoute53HostedZonesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/route53/traffic_policies.go b/resources/services/route53/traffic_policies.go index 844c1eca..6c52111d 100644 --- a/resources/services/route53/traffic_policies.go +++ b/resources/services/route53/traffic_policies.go @@ -3,13 +3,14 @@ package route53 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/route53" "github.com/aws/aws-sdk-go-v2/service/route53/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func TrafficPolicies() *schema.Table { @@ -58,6 +59,7 @@ func fetchRoute53TrafficPolicies(ctx context.Context, meta schema.ClientMeta, pa } return nil } + func fetchRoute53TrafficPolicyVersions(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { r := parent.Item.(types.TrafficPolicySummary) config := route53.ListTrafficPolicyVersionsInput{Id: r.Id} @@ -79,6 +81,7 @@ func fetchRoute53TrafficPolicyVersions(ctx context.Context, meta schema.ClientMe } return nil } + func resolveTrafficPolicyArn() schema.ColumnResolver { return client.ResolveARNGlobal(client.Route53Service, func(resource *schema.Resource) ([]string, error) { return []string{"trafficpolicy", *resource.Item.(types.TrafficPolicySummary).Id}, nil diff --git a/resources/services/route53/traffic_policies_mock_test.go b/resources/services/route53/traffic_policies_mock_test.go index 362a495a..4a3bb0ae 100644 --- a/resources/services/route53/traffic_policies_mock_test.go +++ b/resources/services/route53/traffic_policies_mock_test.go @@ -3,13 +3,14 @@ package route53 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/route53" route53Types "github.com/aws/aws-sdk-go-v2/service/route53/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRoute53TrafficPoliciesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/route53/traffic_policy_versions.go b/resources/services/route53/traffic_policy_versions.go index 5fda5731..aac4da5c 100644 --- a/resources/services/route53/traffic_policy_versions.go +++ b/resources/services/route53/traffic_policy_versions.go @@ -1,12 +1,13 @@ package route53 import ( - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/route53/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func trafficPolicyVersions() *schema.Table { diff --git a/resources/services/route53recoverycontrolconfig/clusters.go b/resources/services/route53recoverycontrolconfig/clusters.go index 8aad0f86..ceff1291 100644 --- a/resources/services/route53recoverycontrolconfig/clusters.go +++ b/resources/services/route53recoverycontrolconfig/clusters.go @@ -3,12 +3,13 @@ package route53recoverycontrolconfig import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/route53recoverycontrolconfig" "github.com/aws/aws-sdk-go-v2/service/route53recoverycontrolconfig/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Clusters() *schema.Table { diff --git a/resources/services/route53recoverycontrolconfig/clusters_mock_test.go b/resources/services/route53recoverycontrolconfig/clusters_mock_test.go index 4346c40f..2dbf0b6a 100644 --- a/resources/services/route53recoverycontrolconfig/clusters_mock_test.go +++ b/resources/services/route53recoverycontrolconfig/clusters_mock_test.go @@ -3,13 +3,14 @@ package route53recoverycontrolconfig import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/route53recoverycontrolconfig" "github.com/aws/aws-sdk-go-v2/service/route53recoverycontrolconfig/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildClusters(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/route53recoverycontrolconfig/control_panel.go b/resources/services/route53recoverycontrolconfig/control_panel.go index 1191b77e..2b197be5 100644 --- a/resources/services/route53recoverycontrolconfig/control_panel.go +++ b/resources/services/route53recoverycontrolconfig/control_panel.go @@ -3,12 +3,13 @@ package route53recoverycontrolconfig import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/route53recoverycontrolconfig" "github.com/aws/aws-sdk-go-v2/service/route53recoverycontrolconfig/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ControlPanels() *schema.Table { diff --git a/resources/services/route53recoverycontrolconfig/control_panel_mock_test.go b/resources/services/route53recoverycontrolconfig/control_panel_mock_test.go index 41155170..7fe5b398 100644 --- a/resources/services/route53recoverycontrolconfig/control_panel_mock_test.go +++ b/resources/services/route53recoverycontrolconfig/control_panel_mock_test.go @@ -3,13 +3,14 @@ package route53recoverycontrolconfig import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/route53recoverycontrolconfig" "github.com/aws/aws-sdk-go-v2/service/route53recoverycontrolconfig/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildControlPanels(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/route53recoverycontrolconfig/routing_controls.go b/resources/services/route53recoverycontrolconfig/routing_controls.go index b764471a..d02da10f 100644 --- a/resources/services/route53recoverycontrolconfig/routing_controls.go +++ b/resources/services/route53recoverycontrolconfig/routing_controls.go @@ -3,12 +3,13 @@ package route53recoverycontrolconfig import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/route53recoverycontrolconfig" "github.com/aws/aws-sdk-go-v2/service/route53recoverycontrolconfig/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func routingControls() *schema.Table { diff --git a/resources/services/route53recoverycontrolconfig/safety_rules.go b/resources/services/route53recoverycontrolconfig/safety_rules.go index ef71cb85..aacb421f 100644 --- a/resources/services/route53recoverycontrolconfig/safety_rules.go +++ b/resources/services/route53recoverycontrolconfig/safety_rules.go @@ -3,12 +3,13 @@ package route53recoverycontrolconfig import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/route53recoverycontrolconfig" "github.com/aws/aws-sdk-go-v2/service/route53recoverycontrolconfig/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func safetyRules() *schema.Table { diff --git a/resources/services/route53recoveryreadiness/cells.go b/resources/services/route53recoveryreadiness/cells.go index e5d58512..0786216c 100644 --- a/resources/services/route53recoveryreadiness/cells.go +++ b/resources/services/route53recoveryreadiness/cells.go @@ -3,12 +3,13 @@ package route53recoveryreadiness import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/route53recoveryreadiness" "github.com/aws/aws-sdk-go-v2/service/route53recoveryreadiness/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Cells() *schema.Table { diff --git a/resources/services/route53recoveryreadiness/cells_mock_test.go b/resources/services/route53recoveryreadiness/cells_mock_test.go index 17ec112c..2b1fe94f 100644 --- a/resources/services/route53recoveryreadiness/cells_mock_test.go +++ b/resources/services/route53recoveryreadiness/cells_mock_test.go @@ -3,13 +3,14 @@ package route53recoveryreadiness import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/route53recoveryreadiness" "github.com/aws/aws-sdk-go-v2/service/route53recoveryreadiness/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildCells(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/route53recoveryreadiness/readiness_checks.go b/resources/services/route53recoveryreadiness/readiness_checks.go index cfccd780..d18998e6 100644 --- a/resources/services/route53recoveryreadiness/readiness_checks.go +++ b/resources/services/route53recoveryreadiness/readiness_checks.go @@ -3,12 +3,13 @@ package route53recoveryreadiness import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/route53recoveryreadiness" "github.com/aws/aws-sdk-go-v2/service/route53recoveryreadiness/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ReadinessChecks() *schema.Table { diff --git a/resources/services/route53recoveryreadiness/readiness_checks_mock_test.go b/resources/services/route53recoveryreadiness/readiness_checks_mock_test.go index 9878e2a5..6940fcc5 100644 --- a/resources/services/route53recoveryreadiness/readiness_checks_mock_test.go +++ b/resources/services/route53recoveryreadiness/readiness_checks_mock_test.go @@ -3,13 +3,14 @@ package route53recoveryreadiness import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/route53recoveryreadiness" "github.com/aws/aws-sdk-go-v2/service/route53recoveryreadiness/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildReadinessChecks(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/route53recoveryreadiness/recovery_groups.go b/resources/services/route53recoveryreadiness/recovery_groups.go index dbf3a3d8..f86a1353 100644 --- a/resources/services/route53recoveryreadiness/recovery_groups.go +++ b/resources/services/route53recoveryreadiness/recovery_groups.go @@ -3,12 +3,13 @@ package route53recoveryreadiness import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/route53recoveryreadiness" "github.com/aws/aws-sdk-go-v2/service/route53recoveryreadiness/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func RecoveryGroups() *schema.Table { diff --git a/resources/services/route53recoveryreadiness/recovery_groups_mock_test.go b/resources/services/route53recoveryreadiness/recovery_groups_mock_test.go index e25323db..63265341 100644 --- a/resources/services/route53recoveryreadiness/recovery_groups_mock_test.go +++ b/resources/services/route53recoveryreadiness/recovery_groups_mock_test.go @@ -3,13 +3,14 @@ package route53recoveryreadiness import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/route53recoveryreadiness" "github.com/aws/aws-sdk-go-v2/service/route53recoveryreadiness/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRecoveryGroups(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/route53recoveryreadiness/resource_sets.go b/resources/services/route53recoveryreadiness/resource_sets.go index 55eab2af..e79f9346 100644 --- a/resources/services/route53recoveryreadiness/resource_sets.go +++ b/resources/services/route53recoveryreadiness/resource_sets.go @@ -3,12 +3,13 @@ package route53recoveryreadiness import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/route53recoveryreadiness" "github.com/aws/aws-sdk-go-v2/service/route53recoveryreadiness/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ResourceSets() *schema.Table { diff --git a/resources/services/route53recoveryreadiness/resource_sets_mock_test.go b/resources/services/route53recoveryreadiness/resource_sets_mock_test.go index ed4d4a39..c29a5944 100644 --- a/resources/services/route53recoveryreadiness/resource_sets_mock_test.go +++ b/resources/services/route53recoveryreadiness/resource_sets_mock_test.go @@ -3,13 +3,14 @@ package route53recoveryreadiness import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/route53recoveryreadiness" "github.com/aws/aws-sdk-go-v2/service/route53recoveryreadiness/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildResourceSets(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/route53resolver/firewall_domain_list.go b/resources/services/route53resolver/firewall_domain_list.go index b76d1a82..bedc678f 100644 --- a/resources/services/route53resolver/firewall_domain_list.go +++ b/resources/services/route53resolver/firewall_domain_list.go @@ -3,12 +3,13 @@ package route53resolver import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/route53resolver" "github.com/aws/aws-sdk-go-v2/service/route53resolver/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func FirewallDomainLists() *schema.Table { @@ -50,6 +51,7 @@ func fetchFirewallDomainList(ctx context.Context, meta schema.ClientMeta, parent } return nil } + func getFirewallDomainList(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource) error { cl := meta.(*client.Client) svc := cl.Services(client.AWSServiceRoute53resolver).Route53resolver diff --git a/resources/services/route53resolver/firewall_domain_list_mock_test.go b/resources/services/route53resolver/firewall_domain_list_mock_test.go index 2a99e7bc..d9409358 100644 --- a/resources/services/route53resolver/firewall_domain_list_mock_test.go +++ b/resources/services/route53resolver/firewall_domain_list_mock_test.go @@ -3,13 +3,14 @@ package route53resolver import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/route53resolver" "github.com/aws/aws-sdk-go-v2/service/route53resolver/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildFirewallDomainListMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -34,6 +35,7 @@ func buildFirewallDomainListMock(t *testing.T, ctrl *gomock.Controller) client.S Route53resolver: m, } } + func TestFirewallDomainList(t *testing.T) { client.AwsMockTestHelper(t, FirewallDomainLists(), buildFirewallDomainListMock, client.TestOptions{}) } diff --git a/resources/services/route53resolver/firewall_rule_group_associations.go b/resources/services/route53resolver/firewall_rule_group_associations.go index cfaeca4d..66e52418 100644 --- a/resources/services/route53resolver/firewall_rule_group_associations.go +++ b/resources/services/route53resolver/firewall_rule_group_associations.go @@ -3,12 +3,13 @@ package route53resolver import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/route53resolver" "github.com/aws/aws-sdk-go-v2/service/route53resolver/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func FirewallRuleGroupAssociations() *schema.Table { diff --git a/resources/services/route53resolver/firewall_rule_group_associations_mock_test.go b/resources/services/route53resolver/firewall_rule_group_associations_mock_test.go index 47e1e361..ebc3709b 100644 --- a/resources/services/route53resolver/firewall_rule_group_associations_mock_test.go +++ b/resources/services/route53resolver/firewall_rule_group_associations_mock_test.go @@ -3,13 +3,14 @@ package route53resolver import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/route53resolver" "github.com/aws/aws-sdk-go-v2/service/route53resolver/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildFirewallRuleGroupAssociationsMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -29,6 +30,7 @@ func buildFirewallRuleGroupAssociationsMock(t *testing.T, ctrl *gomock.Controlle Route53resolver: m, } } + func TestFirewallRuleGroupAssociations(t *testing.T) { client.AwsMockTestHelper(t, FirewallRuleGroupAssociations(), buildFirewallRuleGroupAssociationsMock, client.TestOptions{}) } diff --git a/resources/services/route53resolver/firewall_rule_group_mock_test.go b/resources/services/route53resolver/firewall_rule_group_mock_test.go index a92ba0ee..f3e2fc75 100644 --- a/resources/services/route53resolver/firewall_rule_group_mock_test.go +++ b/resources/services/route53resolver/firewall_rule_group_mock_test.go @@ -3,13 +3,14 @@ package route53resolver import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/route53resolver" "github.com/aws/aws-sdk-go-v2/service/route53resolver/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildFirewallRuleGroupsMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -34,6 +35,7 @@ func buildFirewallRuleGroupsMock(t *testing.T, ctrl *gomock.Controller) client.S Route53resolver: m, } } + func TestFirewallRuleGroups(t *testing.T) { client.AwsMockTestHelper(t, FirewallRuleGroups(), buildFirewallRuleGroupsMock, client.TestOptions{}) } diff --git a/resources/services/route53resolver/firewall_rule_groups.go b/resources/services/route53resolver/firewall_rule_groups.go index becd32c6..7ac1b78e 100644 --- a/resources/services/route53resolver/firewall_rule_groups.go +++ b/resources/services/route53resolver/firewall_rule_groups.go @@ -3,12 +3,13 @@ package route53resolver import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/route53resolver" "github.com/aws/aws-sdk-go-v2/service/route53resolver/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func FirewallRuleGroups() *schema.Table { diff --git a/resources/services/route53resolver/resolver_endpoints.go b/resources/services/route53resolver/resolver_endpoints.go index 5b0b6f80..067c3fe6 100644 --- a/resources/services/route53resolver/resolver_endpoints.go +++ b/resources/services/route53resolver/resolver_endpoints.go @@ -3,12 +3,13 @@ package route53resolver import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/route53resolver" "github.com/aws/aws-sdk-go-v2/service/route53resolver/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ResolverEndpoints() *schema.Table { diff --git a/resources/services/route53resolver/resolver_endpoints_mock_test.go b/resources/services/route53resolver/resolver_endpoints_mock_test.go index 67ff3540..680beb15 100644 --- a/resources/services/route53resolver/resolver_endpoints_mock_test.go +++ b/resources/services/route53resolver/resolver_endpoints_mock_test.go @@ -3,13 +3,14 @@ package route53resolver import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/route53resolver" "github.com/aws/aws-sdk-go-v2/service/route53resolver/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEndpointsMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -26,6 +27,7 @@ func buildEndpointsMock(t *testing.T, ctrl *gomock.Controller) client.Services { Route53resolver: m, } } + func TestEndpoints(t *testing.T) { client.AwsMockTestHelper(t, ResolverEndpoints(), buildEndpointsMock, client.TestOptions{}) } diff --git a/resources/services/route53resolver/resolver_query_log_config_associations.go b/resources/services/route53resolver/resolver_query_log_config_associations.go index c2df2e6d..ba6fe523 100644 --- a/resources/services/route53resolver/resolver_query_log_config_associations.go +++ b/resources/services/route53resolver/resolver_query_log_config_associations.go @@ -3,11 +3,12 @@ package route53resolver import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/service/route53resolver" "github.com/aws/aws-sdk-go-v2/service/route53resolver/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ResolverQueryLogConfigAssociations() *schema.Table { diff --git a/resources/services/route53resolver/resolver_query_log_config_associations_mock_test.go b/resources/services/route53resolver/resolver_query_log_config_associations_mock_test.go index 8f2fd15d..603433e5 100644 --- a/resources/services/route53resolver/resolver_query_log_config_associations_mock_test.go +++ b/resources/services/route53resolver/resolver_query_log_config_associations_mock_test.go @@ -3,13 +3,14 @@ package route53resolver import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/route53resolver" "github.com/aws/aws-sdk-go-v2/service/route53resolver/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildResolverQueryLogConfigAssociationsMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -26,6 +27,7 @@ func buildResolverQueryLogConfigAssociationsMock(t *testing.T, ctrl *gomock.Cont Route53resolver: m, } } + func TestResolverQueryLogConfigAssociations(t *testing.T) { client.AwsMockTestHelper(t, ResolverQueryLogConfigAssociations(), buildResolverQueryLogConfigAssociationsMock, client.TestOptions{}) } diff --git a/resources/services/route53resolver/resolver_query_log_configs.go b/resources/services/route53resolver/resolver_query_log_configs.go index 54fdfc28..56fbe56c 100644 --- a/resources/services/route53resolver/resolver_query_log_configs.go +++ b/resources/services/route53resolver/resolver_query_log_configs.go @@ -3,12 +3,13 @@ package route53resolver import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/route53resolver" "github.com/aws/aws-sdk-go-v2/service/route53resolver/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ResolverQueryLogConfigs() *schema.Table { diff --git a/resources/services/route53resolver/resolver_query_log_configs_mock_test.go b/resources/services/route53resolver/resolver_query_log_configs_mock_test.go index 0caf7438..d0cbb659 100644 --- a/resources/services/route53resolver/resolver_query_log_configs_mock_test.go +++ b/resources/services/route53resolver/resolver_query_log_configs_mock_test.go @@ -3,13 +3,14 @@ package route53resolver import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/route53resolver" "github.com/aws/aws-sdk-go-v2/service/route53resolver/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildResolverQueryLogConfigsMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -26,6 +27,7 @@ func buildResolverQueryLogConfigsMock(t *testing.T, ctrl *gomock.Controller) cli Route53resolver: m, } } + func TestResolverQueryLogConfigs(t *testing.T) { client.AwsMockTestHelper(t, ResolverQueryLogConfigs(), buildResolverQueryLogConfigsMock, client.TestOptions{}) } diff --git a/resources/services/route53resolver/resolver_rule_associations.go b/resources/services/route53resolver/resolver_rule_associations.go index 739c48d1..cdb90514 100644 --- a/resources/services/route53resolver/resolver_rule_associations.go +++ b/resources/services/route53resolver/resolver_rule_associations.go @@ -3,11 +3,12 @@ package route53resolver import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/service/route53resolver" "github.com/aws/aws-sdk-go-v2/service/route53resolver/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ResolverRuleAssociations() *schema.Table { diff --git a/resources/services/route53resolver/resolver_rule_associations_mock_test.go b/resources/services/route53resolver/resolver_rule_associations_mock_test.go index 213348af..7da58918 100644 --- a/resources/services/route53resolver/resolver_rule_associations_mock_test.go +++ b/resources/services/route53resolver/resolver_rule_associations_mock_test.go @@ -3,13 +3,14 @@ package route53resolver import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/route53resolver" "github.com/aws/aws-sdk-go-v2/service/route53resolver/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildResolverRuleAssociationsMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -26,6 +27,7 @@ func buildResolverRuleAssociationsMock(t *testing.T, ctrl *gomock.Controller) cl Route53resolver: m, } } + func TestResolverRuleAssociations(t *testing.T) { client.AwsMockTestHelper(t, ResolverRuleAssociations(), buildResolverRuleAssociationsMock, client.TestOptions{}) } diff --git a/resources/services/route53resolver/resolver_rules.go b/resources/services/route53resolver/resolver_rules.go index 81bf8620..70d3efdb 100644 --- a/resources/services/route53resolver/resolver_rules.go +++ b/resources/services/route53resolver/resolver_rules.go @@ -3,12 +3,13 @@ package route53resolver import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/route53resolver" "github.com/aws/aws-sdk-go-v2/service/route53resolver/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ResolverRules() *schema.Table { diff --git a/resources/services/route53resolver/resolver_rules_mock_test.go b/resources/services/route53resolver/resolver_rules_mock_test.go index f1065516..04128484 100644 --- a/resources/services/route53resolver/resolver_rules_mock_test.go +++ b/resources/services/route53resolver/resolver_rules_mock_test.go @@ -3,13 +3,14 @@ package route53resolver import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/route53resolver" "github.com/aws/aws-sdk-go-v2/service/route53resolver/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildResolverRulesMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -26,6 +27,7 @@ func buildResolverRulesMock(t *testing.T, ctrl *gomock.Controller) client.Servic Route53resolver: m, } } + func TestResolverRules(t *testing.T) { client.AwsMockTestHelper(t, ResolverRules(), buildResolverRulesMock, client.TestOptions{}) } diff --git a/resources/services/s3/access_grant_instances.go b/resources/services/s3/access_grant_instances.go index a8070b9e..f9586099 100644 --- a/resources/services/s3/access_grant_instances.go +++ b/resources/services/s3/access_grant_instances.go @@ -3,13 +3,14 @@ package s3 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3control" "github.com/aws/aws-sdk-go-v2/service/s3control/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func AccessGrantInstances() *schema.Table { diff --git a/resources/services/s3/access_grant_instances_mock_test.go b/resources/services/s3/access_grant_instances_mock_test.go index 799c6d85..9c67557c 100644 --- a/resources/services/s3/access_grant_instances_mock_test.go +++ b/resources/services/s3/access_grant_instances_mock_test.go @@ -3,13 +3,14 @@ package s3 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/s3control" "github.com/aws/aws-sdk-go-v2/service/s3control/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildS3AccessGrantInstances(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/s3/access_grants.go b/resources/services/s3/access_grants.go index 170577c1..8add4aa6 100644 --- a/resources/services/s3/access_grants.go +++ b/resources/services/s3/access_grants.go @@ -3,13 +3,14 @@ package s3 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3control" "github.com/aws/aws-sdk-go-v2/service/s3control/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func AccessGrants() *schema.Table { diff --git a/resources/services/s3/access_grants_mock_test.go b/resources/services/s3/access_grants_mock_test.go index 478b18d9..f9d1f7ed 100644 --- a/resources/services/s3/access_grants_mock_test.go +++ b/resources/services/s3/access_grants_mock_test.go @@ -3,13 +3,14 @@ package s3 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/s3control" "github.com/aws/aws-sdk-go-v2/service/s3control/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildS3AccessGrants(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/s3/access_points.go b/resources/services/s3/access_points.go index bc79a4ef..fcf8c6ae 100644 --- a/resources/services/s3/access_points.go +++ b/resources/services/s3/access_points.go @@ -3,13 +3,14 @@ package s3 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3control" "github.com/aws/aws-sdk-go-v2/service/s3control/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func AccessPoints() *schema.Table { diff --git a/resources/services/s3/access_points_mock_test.go b/resources/services/s3/access_points_mock_test.go index 32fabbf0..f6339789 100644 --- a/resources/services/s3/access_points_mock_test.go +++ b/resources/services/s3/access_points_mock_test.go @@ -3,13 +3,14 @@ package s3 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/s3control" "github.com/aws/aws-sdk-go-v2/service/s3control/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildS3AccessPoints(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/s3/bucket_cors_rules.go b/resources/services/s3/bucket_cors_rules.go index 82d56614..14c0606f 100644 --- a/resources/services/s3/bucket_cors_rules.go +++ b/resources/services/s3/bucket_cors_rules.go @@ -3,14 +3,16 @@ package s3 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/ohaws" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/cloudquery/plugin-sdk/v4/scalar" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/ohaws" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/scalar" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func bucketCorsRules() *schema.Table { @@ -30,6 +32,7 @@ func bucketCorsRules() *schema.Table { }, } } + func fetchS3BucketCorsRules(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { r := parent.Item.(*ohaws.WrappedBucket) cl := meta.(*client.Client) diff --git a/resources/services/s3/bucket_encryption_rules.go b/resources/services/s3/bucket_encryption_rules.go index 8439c81d..554e42f0 100644 --- a/resources/services/s3/bucket_encryption_rules.go +++ b/resources/services/s3/bucket_encryption_rules.go @@ -3,14 +3,16 @@ package s3 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/ohaws" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/cloudquery/plugin-sdk/v4/scalar" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/ohaws" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/scalar" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func bucketEncryptionRules() *schema.Table { diff --git a/resources/services/s3/bucket_grants.go b/resources/services/s3/bucket_grants.go index e0e6a275..1c5123e2 100644 --- a/resources/services/s3/bucket_grants.go +++ b/resources/services/s3/bucket_grants.go @@ -4,14 +4,16 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/ohaws" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/cloudquery/plugin-sdk/v4/scalar" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/ohaws" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/scalar" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func bucketGrants() *schema.Table { @@ -49,6 +51,7 @@ func bucketGrants() *schema.Table { }, } } + func fetchS3BucketGrants(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { r := parent.Item.(*ohaws.WrappedBucket) cl := meta.(*client.Client) diff --git a/resources/services/s3/bucket_lifecycles.go b/resources/services/s3/bucket_lifecycles.go index 8882b91a..1e3661f0 100644 --- a/resources/services/s3/bucket_lifecycles.go +++ b/resources/services/s3/bucket_lifecycles.go @@ -3,14 +3,16 @@ package s3 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/ohaws" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/cloudquery/plugin-sdk/v4/scalar" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/ohaws" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/scalar" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func bucketLifecycles() *schema.Table { diff --git a/resources/services/s3/bucket_logging.go b/resources/services/s3/bucket_logging.go index 6e3324b7..8a283aa3 100644 --- a/resources/services/s3/bucket_logging.go +++ b/resources/services/s3/bucket_logging.go @@ -3,12 +3,14 @@ package s3 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/ohaws" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/ohaws" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func bucketLogging() *schema.Table { @@ -28,6 +30,7 @@ func bucketLogging() *schema.Table { }, } } + func fetchBucketLogging(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { r := parent.Item.(*ohaws.WrappedBucket) cl := meta.(*client.Client) diff --git a/resources/services/s3/bucket_notification_configurations.go b/resources/services/s3/bucket_notification_configurations.go index 8b701df1..ffdbda76 100644 --- a/resources/services/s3/bucket_notification_configurations.go +++ b/resources/services/s3/bucket_notification_configurations.go @@ -3,14 +3,16 @@ package s3 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/ohaws" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/cloudquery/plugin-sdk/v4/scalar" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/ohaws" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/scalar" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func bucketNotificationConfigurations() *schema.Table { diff --git a/resources/services/s3/bucket_object_lock_configurations.go b/resources/services/s3/bucket_object_lock_configurations.go index 8147345d..2eade371 100644 --- a/resources/services/s3/bucket_object_lock_configurations.go +++ b/resources/services/s3/bucket_object_lock_configurations.go @@ -3,14 +3,16 @@ package s3 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/ohaws" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/cloudquery/plugin-sdk/v4/scalar" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/ohaws" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/scalar" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func bucketObjectLockConfigurations() *schema.Table { diff --git a/resources/services/s3/bucket_ownership_controls.go b/resources/services/s3/bucket_ownership_controls.go index 26ef842b..68523d5f 100644 --- a/resources/services/s3/bucket_ownership_controls.go +++ b/resources/services/s3/bucket_ownership_controls.go @@ -3,13 +3,15 @@ package s3 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/ohaws" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/ohaws" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func bucketOwnershipControls() *schema.Table { @@ -29,6 +31,7 @@ func bucketOwnershipControls() *schema.Table { }, } } + func fetchBucketOwnershipControls(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { r := parent.Item.(*ohaws.WrappedBucket) cl := meta.(*client.Client) diff --git a/resources/services/s3/bucket_policies.go b/resources/services/s3/bucket_policies.go index 789cb9dc..28167037 100644 --- a/resources/services/s3/bucket_policies.go +++ b/resources/services/s3/bucket_policies.go @@ -3,13 +3,15 @@ package s3 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/ohaws" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/ohaws" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func bucketPolicies() *schema.Table { @@ -34,6 +36,7 @@ func bucketPolicies() *schema.Table { }, } } + func fetchS3BucketPolicies(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { r := parent.Item.(*ohaws.WrappedBucket) cl := meta.(*client.Client) diff --git a/resources/services/s3/bucket_public_access_block.go b/resources/services/s3/bucket_public_access_block.go index 17051fd1..2eca2bfd 100644 --- a/resources/services/s3/bucket_public_access_block.go +++ b/resources/services/s3/bucket_public_access_block.go @@ -3,12 +3,14 @@ package s3 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/ohaws" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/ohaws" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func bucketPublicAccessBlock() *schema.Table { @@ -28,6 +30,7 @@ func bucketPublicAccessBlock() *schema.Table { }, } } + func fetchBucketPublicAccessBlock(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { r := parent.Item.(*ohaws.WrappedBucket) cl := meta.(*client.Client) diff --git a/resources/services/s3/bucket_replications.go b/resources/services/s3/bucket_replications.go index 500114f2..a7f46689 100644 --- a/resources/services/s3/bucket_replications.go +++ b/resources/services/s3/bucket_replications.go @@ -3,12 +3,14 @@ package s3 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/ohaws" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/ohaws" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func bucketReplications() *schema.Table { @@ -28,6 +30,7 @@ func bucketReplications() *schema.Table { }, } } + func fetchS3BucketReplications(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { r := parent.Item.(*ohaws.WrappedBucket) cl := meta.(*client.Client) diff --git a/resources/services/s3/bucket_versioning.go b/resources/services/s3/bucket_versioning.go index a9e5905b..870eb93a 100644 --- a/resources/services/s3/bucket_versioning.go +++ b/resources/services/s3/bucket_versioning.go @@ -3,12 +3,14 @@ package s3 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/ohaws" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/ohaws" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func bucketVersionings() *schema.Table { @@ -28,6 +30,7 @@ func bucketVersionings() *schema.Table { }, } } + func fetchBucketVersionings(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { r := parent.Item.(*ohaws.WrappedBucket) cl := meta.(*client.Client) diff --git a/resources/services/s3/bucket_websites.go b/resources/services/s3/bucket_websites.go index 77810196..35d136fc 100644 --- a/resources/services/s3/bucket_websites.go +++ b/resources/services/s3/bucket_websites.go @@ -3,13 +3,15 @@ package s3 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/ohaws" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/cloudquery/plugin-sdk/v4/scalar" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/ohaws" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/scalar" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func bucketWebsites() *schema.Table { @@ -29,6 +31,7 @@ func bucketWebsites() *schema.Table { }, } } + func fetchS3BucketWebsites(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { r := parent.Item.(*ohaws.WrappedBucket) cl := meta.(*client.Client) diff --git a/resources/services/s3/buckets.go b/resources/services/s3/buckets.go index a064c823..3a6d88a2 100644 --- a/resources/services/s3/buckets.go +++ b/resources/services/s3/buckets.go @@ -6,13 +6,15 @@ import ( "errors" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/ohaws" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/ohaws" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Buckets() *schema.Table { diff --git a/resources/services/s3/buckets_mock_test.go b/resources/services/s3/buckets_mock_test.go index ec0d29a1..d62c2bab 100644 --- a/resources/services/s3/buckets_mock_test.go +++ b/resources/services/s3/buckets_mock_test.go @@ -3,13 +3,14 @@ package s3 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/s3" s3Types "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildS3Buckets(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/s3/multi_region_access_points.go b/resources/services/s3/multi_region_access_points.go index 7f58f649..8dfed90c 100644 --- a/resources/services/s3/multi_region_access_points.go +++ b/resources/services/s3/multi_region_access_points.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/s3control" "github.com/aws/aws-sdk-go-v2/service/s3control/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func MultiRegionAccessPoints() *schema.Table { diff --git a/resources/services/s3/multi_region_access_points_mock_test.go b/resources/services/s3/multi_region_access_points_mock_test.go index 8a6a87f7..e34269e1 100644 --- a/resources/services/s3/multi_region_access_points_mock_test.go +++ b/resources/services/s3/multi_region_access_points_mock_test.go @@ -3,13 +3,14 @@ package s3 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/s3control" "github.com/aws/aws-sdk-go-v2/service/s3control/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildS3MultiRegionAccessPoints(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/sagemaker/apps.go b/resources/services/sagemaker/apps.go index 6cf4f053..555f984c 100644 --- a/resources/services/sagemaker/apps.go +++ b/resources/services/sagemaker/apps.go @@ -3,13 +3,14 @@ package sagemaker import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/sagemaker" "github.com/aws/aws-sdk-go-v2/service/sagemaker/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Apps() *schema.Table { @@ -40,6 +41,7 @@ func Apps() *schema.Table { }, } } + func fetchSagemakerApps(ctx context.Context, meta schema.ClientMeta, _ *schema.Resource, res chan<- any) error { cl := meta.(*client.Client) svc := cl.Services(client.AWSServiceSagemaker).Sagemaker diff --git a/resources/services/sagemaker/apps_mock_test.go b/resources/services/sagemaker/apps_mock_test.go index 9e13569a..9fe52876 100644 --- a/resources/services/sagemaker/apps_mock_test.go +++ b/resources/services/sagemaker/apps_mock_test.go @@ -3,13 +3,14 @@ package sagemaker import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/sagemaker" types "github.com/aws/aws-sdk-go-v2/service/sagemaker/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildSageMakerApps(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/sagemaker/endpoint_configurations.go b/resources/services/sagemaker/endpoint_configurations.go index fb14b3b7..14067e7c 100644 --- a/resources/services/sagemaker/endpoint_configurations.go +++ b/resources/services/sagemaker/endpoint_configurations.go @@ -3,13 +3,14 @@ package sagemaker import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/sagemaker" "github.com/aws/aws-sdk-go-v2/service/sagemaker/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func EndpointConfigurations() *schema.Table { diff --git a/resources/services/sagemaker/endpoint_configurations_mock_test.go b/resources/services/sagemaker/endpoint_configurations_mock_test.go index 962a4451..cc0f469c 100644 --- a/resources/services/sagemaker/endpoint_configurations_mock_test.go +++ b/resources/services/sagemaker/endpoint_configurations_mock_test.go @@ -3,13 +3,14 @@ package sagemaker import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/sagemaker" types "github.com/aws/aws-sdk-go-v2/service/sagemaker/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildSageMakerEndpointConfigs(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/sagemaker/models.go b/resources/services/sagemaker/models.go index 30f3c4a7..e9259a2d 100644 --- a/resources/services/sagemaker/models.go +++ b/resources/services/sagemaker/models.go @@ -3,13 +3,14 @@ package sagemaker import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/sagemaker" "github.com/aws/aws-sdk-go-v2/service/sagemaker/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Models() *schema.Table { diff --git a/resources/services/sagemaker/models_mock_test.go b/resources/services/sagemaker/models_mock_test.go index 1646a93c..dbf3028e 100644 --- a/resources/services/sagemaker/models_mock_test.go +++ b/resources/services/sagemaker/models_mock_test.go @@ -3,13 +3,14 @@ package sagemaker import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/sagemaker" types "github.com/aws/aws-sdk-go-v2/service/sagemaker/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildSageMakerModels(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/sagemaker/notebook_instances.go b/resources/services/sagemaker/notebook_instances.go index 500cc229..4d1b14b3 100644 --- a/resources/services/sagemaker/notebook_instances.go +++ b/resources/services/sagemaker/notebook_instances.go @@ -3,13 +3,14 @@ package sagemaker import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/sagemaker" "github.com/aws/aws-sdk-go-v2/service/sagemaker/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func NotebookInstances() *schema.Table { diff --git a/resources/services/sagemaker/notebook_instances_mock_test.go b/resources/services/sagemaker/notebook_instances_mock_test.go index 2ef15f50..eca59174 100644 --- a/resources/services/sagemaker/notebook_instances_mock_test.go +++ b/resources/services/sagemaker/notebook_instances_mock_test.go @@ -3,13 +3,14 @@ package sagemaker import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/sagemaker" types "github.com/aws/aws-sdk-go-v2/service/sagemaker/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildSageMakerNotebookInstances(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/sagemaker/training_jobs.go b/resources/services/sagemaker/training_jobs.go index e1b146d8..0a3f33b4 100644 --- a/resources/services/sagemaker/training_jobs.go +++ b/resources/services/sagemaker/training_jobs.go @@ -3,13 +3,14 @@ package sagemaker import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/sagemaker" "github.com/aws/aws-sdk-go-v2/service/sagemaker/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func TrainingJobs() *schema.Table { diff --git a/resources/services/sagemaker/training_jobs_mock_test.go b/resources/services/sagemaker/training_jobs_mock_test.go index d4075b9c..6eaafb9a 100644 --- a/resources/services/sagemaker/training_jobs_mock_test.go +++ b/resources/services/sagemaker/training_jobs_mock_test.go @@ -3,13 +3,14 @@ package sagemaker import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/sagemaker" types "github.com/aws/aws-sdk-go-v2/service/sagemaker/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildSageMakerTrainingJobs(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/savingsplans/savingsplans.go b/resources/services/savingsplans/savingsplans.go index 4cb88188..dd00b8d7 100644 --- a/resources/services/savingsplans/savingsplans.go +++ b/resources/services/savingsplans/savingsplans.go @@ -3,13 +3,14 @@ package savingsplans import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/savingsplans" "github.com/aws/aws-sdk-go-v2/service/savingsplans/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Plans() *schema.Table { diff --git a/resources/services/savingsplans/savingsplans_mock_test.go b/resources/services/savingsplans/savingsplans_mock_test.go index aca7e1bd..8006e732 100644 --- a/resources/services/savingsplans/savingsplans_mock_test.go +++ b/resources/services/savingsplans/savingsplans_mock_test.go @@ -3,14 +3,15 @@ package savingsplans import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/savingsplans" "github.com/aws/aws-sdk-go-v2/service/savingsplans/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildSavingPlansPlans(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/scheduler/schedule_groups.go b/resources/services/scheduler/schedule_groups.go index 64e4fcd6..0f633cd7 100644 --- a/resources/services/scheduler/schedule_groups.go +++ b/resources/services/scheduler/schedule_groups.go @@ -3,14 +3,15 @@ package scheduler import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/scheduler" "github.com/aws/aws-sdk-go-v2/service/scheduler/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func ScheduleGroups() *schema.Table { diff --git a/resources/services/scheduler/schedule_groups_mock_test.go b/resources/services/scheduler/schedule_groups_mock_test.go index fa505059..bb6356cb 100644 --- a/resources/services/scheduler/schedule_groups_mock_test.go +++ b/resources/services/scheduler/schedule_groups_mock_test.go @@ -3,13 +3,14 @@ package scheduler import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/scheduler" "github.com/aws/aws-sdk-go-v2/service/scheduler/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildSchedulerScheduleGroupsMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -29,6 +30,7 @@ func buildSchedulerScheduleGroupsMock(t *testing.T, ctrl *gomock.Controller) cli Scheduler: m, } } + func TestSchedulerSchedulerGroups(t *testing.T) { client.AwsMockTestHelper(t, ScheduleGroups(), buildSchedulerScheduleGroupsMock, client.TestOptions{}) } diff --git a/resources/services/scheduler/schedules.go b/resources/services/scheduler/schedules.go index 1aac7799..4815cf5c 100644 --- a/resources/services/scheduler/schedules.go +++ b/resources/services/scheduler/schedules.go @@ -3,15 +3,16 @@ package scheduler import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/scheduler" "github.com/aws/aws-sdk-go-v2/service/scheduler/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" "github.com/thoas/go-funk" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Schedules() *schema.Table { diff --git a/resources/services/scheduler/schedules_mock_test.go b/resources/services/scheduler/schedules_mock_test.go index 21fb596c..976aba4a 100644 --- a/resources/services/scheduler/schedules_mock_test.go +++ b/resources/services/scheduler/schedules_mock_test.go @@ -3,13 +3,14 @@ package scheduler import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/scheduler" "github.com/aws/aws-sdk-go-v2/service/scheduler/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildSchedulerSchedulesMock(t *testing.T, ctrl *gomock.Controller) client.Services { @@ -34,6 +35,7 @@ func buildSchedulerSchedulesMock(t *testing.T, ctrl *gomock.Controller) client.S Scheduler: m, } } + func TestSchedulesMock(t *testing.T) { client.AwsMockTestHelper(t, Schedules(), buildSchedulerSchedulesMock, client.TestOptions{}) } diff --git a/resources/services/secretsmanager/secrets.go b/resources/services/secretsmanager/secrets.go index 452e7073..89104b20 100644 --- a/resources/services/secretsmanager/secrets.go +++ b/resources/services/secretsmanager/secrets.go @@ -4,13 +4,14 @@ import ( "context" "encoding/json" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/secretsmanager" "github.com/aws/aws-sdk-go-v2/service/secretsmanager/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Secrets() *schema.Table { diff --git a/resources/services/secretsmanager/secrets_mock_test.go b/resources/services/secretsmanager/secrets_mock_test.go index f20896f5..338ce24c 100644 --- a/resources/services/secretsmanager/secrets_mock_test.go +++ b/resources/services/secretsmanager/secrets_mock_test.go @@ -3,13 +3,14 @@ package secretsmanager import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/secretsmanager" "github.com/aws/aws-sdk-go-v2/service/secretsmanager/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildSecretsmanagerModels(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/secretsmanager/secrets_version.go b/resources/services/secretsmanager/secrets_version.go index 441c5bc8..b1526779 100644 --- a/resources/services/secretsmanager/secrets_version.go +++ b/resources/services/secretsmanager/secrets_version.go @@ -3,13 +3,14 @@ package secretsmanager import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/secretsmanager" "github.com/aws/aws-sdk-go-v2/service/secretsmanager/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func secretVersions() *schema.Table { diff --git a/resources/services/securityhub/enabled_standards.go b/resources/services/securityhub/enabled_standards.go index d9f86170..a49f6523 100644 --- a/resources/services/securityhub/enabled_standards.go +++ b/resources/services/securityhub/enabled_standards.go @@ -3,13 +3,14 @@ package securityhub import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/securityhub" "github.com/aws/aws-sdk-go-v2/service/securityhub/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func EnabledStandards() *schema.Table { diff --git a/resources/services/securityhub/enabled_standards_mock_test.go b/resources/services/securityhub/enabled_standards_mock_test.go index b83d357f..1380d8df 100644 --- a/resources/services/securityhub/enabled_standards_mock_test.go +++ b/resources/services/securityhub/enabled_standards_mock_test.go @@ -3,14 +3,15 @@ package securityhub import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/securityhub" "github.com/aws/aws-sdk-go-v2/service/securityhub/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildEnabledStandards(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/securityhub/hubs.go b/resources/services/securityhub/hubs.go index 35e7fddd..238f90fe 100644 --- a/resources/services/securityhub/hubs.go +++ b/resources/services/securityhub/hubs.go @@ -3,12 +3,13 @@ package securityhub import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/securityhub" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Hubs() *schema.Table { diff --git a/resources/services/securityhub/hubs_mock_test.go b/resources/services/securityhub/hubs_mock_test.go index 66266b3f..2934399b 100644 --- a/resources/services/securityhub/hubs_mock_test.go +++ b/resources/services/securityhub/hubs_mock_test.go @@ -4,13 +4,14 @@ import ( "testing" "time" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/securityhub" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildHubs(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/servicecatalog/launch_paths.go b/resources/services/servicecatalog/launch_paths.go index 48d6904f..67747236 100644 --- a/resources/services/servicecatalog/launch_paths.go +++ b/resources/services/servicecatalog/launch_paths.go @@ -3,13 +3,14 @@ package servicecatalog import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/servicecatalog" "github.com/aws/aws-sdk-go-v2/service/servicecatalog/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func launchPaths() *schema.Table { @@ -51,6 +52,7 @@ func launchPaths() *schema.Table { }, } } + func listLaunchPaths(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { cl := meta.(*client.Client) svc := cl.Services(client.AWSServiceServicecatalog).Servicecatalog diff --git a/resources/services/servicecatalog/portfolios.go b/resources/services/servicecatalog/portfolios.go index f41322a1..596c9bab 100644 --- a/resources/services/servicecatalog/portfolios.go +++ b/resources/services/servicecatalog/portfolios.go @@ -3,13 +3,14 @@ package servicecatalog import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/servicecatalog" "github.com/aws/aws-sdk-go-v2/service/servicecatalog/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Portfolios() *schema.Table { diff --git a/resources/services/servicecatalog/portfolios_mock_test.go b/resources/services/servicecatalog/portfolios_mock_test.go index 049b0ea4..6a5a37b1 100644 --- a/resources/services/servicecatalog/portfolios_mock_test.go +++ b/resources/services/servicecatalog/portfolios_mock_test.go @@ -3,12 +3,13 @@ package servicecatalog import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/servicecatalog" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildPortfolios(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/servicecatalog/products.go b/resources/services/servicecatalog/products.go index a08914da..7fb1a552 100644 --- a/resources/services/servicecatalog/products.go +++ b/resources/services/servicecatalog/products.go @@ -3,13 +3,14 @@ package servicecatalog import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/servicecatalog" "github.com/aws/aws-sdk-go-v2/service/servicecatalog/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Products() *schema.Table { diff --git a/resources/services/servicecatalog/products_mock_test.go b/resources/services/servicecatalog/products_mock_test.go index da22517c..4d7b7e01 100644 --- a/resources/services/servicecatalog/products_mock_test.go +++ b/resources/services/servicecatalog/products_mock_test.go @@ -3,12 +3,13 @@ package servicecatalog import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/servicecatalog" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildProducts(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/servicecatalog/provisioned_products.go b/resources/services/servicecatalog/provisioned_products.go index 177450f5..65b04f8a 100644 --- a/resources/services/servicecatalog/provisioned_products.go +++ b/resources/services/servicecatalog/provisioned_products.go @@ -3,13 +3,14 @@ package servicecatalog import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/servicecatalog" "github.com/aws/aws-sdk-go-v2/service/servicecatalog/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func ProvisionedProducts() *schema.Table { diff --git a/resources/services/servicecatalog/provisioned_products_mock_test.go b/resources/services/servicecatalog/provisioned_products_mock_test.go index e48e96d4..04adae73 100644 --- a/resources/services/servicecatalog/provisioned_products_mock_test.go +++ b/resources/services/servicecatalog/provisioned_products_mock_test.go @@ -3,12 +3,13 @@ package servicecatalog import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/servicecatalog" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildProvisionedProducts(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/servicecatalog/provisioning_artifact.go b/resources/services/servicecatalog/provisioning_artifact.go index f2f5278b..07680d91 100644 --- a/resources/services/servicecatalog/provisioning_artifact.go +++ b/resources/services/servicecatalog/provisioning_artifact.go @@ -3,12 +3,13 @@ package servicecatalog import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/servicecatalog" "github.com/aws/aws-sdk-go-v2/service/servicecatalog/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func provisioningArtifact() *schema.Table { diff --git a/resources/services/servicecatalog/provisioning_parameters.go b/resources/services/servicecatalog/provisioning_parameters.go index 30bfbfa8..41c29f2d 100644 --- a/resources/services/servicecatalog/provisioning_parameters.go +++ b/resources/services/servicecatalog/provisioning_parameters.go @@ -3,12 +3,13 @@ package servicecatalog import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/servicecatalog" "github.com/aws/aws-sdk-go-v2/service/servicecatalog/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func provisioningParameters() *schema.Table { diff --git a/resources/services/servicediscovery/helpers.go b/resources/services/servicediscovery/helpers.go index adaa06ce..4df8f8c6 100644 --- a/resources/services/servicediscovery/helpers.go +++ b/resources/services/servicediscovery/helpers.go @@ -3,11 +3,11 @@ package servicediscovery import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/service/servicediscovery" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/thoas/go-funk" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" ) func resolveServicediscoveryTags(path string) schema.ColumnResolver { diff --git a/resources/services/servicediscovery/instances.go b/resources/services/servicediscovery/instances.go index 284c9710..1f160fc4 100644 --- a/resources/services/servicediscovery/instances.go +++ b/resources/services/servicediscovery/instances.go @@ -3,12 +3,13 @@ package servicediscovery import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/servicediscovery" "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func instances() *schema.Table { diff --git a/resources/services/servicediscovery/namespaces.go b/resources/services/servicediscovery/namespaces.go index 61e21053..ae188dcb 100644 --- a/resources/services/servicediscovery/namespaces.go +++ b/resources/services/servicediscovery/namespaces.go @@ -3,14 +3,15 @@ package servicediscovery import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/servicediscovery" "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Namespaces() *schema.Table { @@ -40,6 +41,7 @@ func Namespaces() *schema.Table { }, } } + func fetchNamespaces(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { cl := meta.(*client.Client) svc := cl.Services(client.AWSServiceServicediscovery).Servicediscovery diff --git a/resources/services/servicediscovery/namespaces_test.go b/resources/services/servicediscovery/namespaces_test.go index ecb9c323..57f2a739 100644 --- a/resources/services/servicediscovery/namespaces_test.go +++ b/resources/services/servicediscovery/namespaces_test.go @@ -3,14 +3,15 @@ package servicediscovery import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/servicediscovery" "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildNamespaces(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/servicediscovery/services.go b/resources/services/servicediscovery/services.go index 6d410659..bbf4549a 100644 --- a/resources/services/servicediscovery/services.go +++ b/resources/services/servicediscovery/services.go @@ -3,14 +3,15 @@ package servicediscovery import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/servicediscovery" "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Services() *schema.Table { @@ -43,6 +44,7 @@ func Services() *schema.Table { }, } } + func fetchServices(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { cl := meta.(*client.Client) svc := cl.Services(client.AWSServiceServicediscovery).Servicediscovery diff --git a/resources/services/servicediscovery/services_test.go b/resources/services/servicediscovery/services_test.go index e87f034b..68f7df20 100644 --- a/resources/services/servicediscovery/services_test.go +++ b/resources/services/servicediscovery/services_test.go @@ -3,14 +3,15 @@ package servicediscovery import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/servicediscovery" "github.com/aws/aws-sdk-go-v2/service/servicediscovery/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildServices(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ses/configuration_set_event_destinations.go b/resources/services/ses/configuration_set_event_destinations.go index 34188b26..052de502 100644 --- a/resources/services/ses/configuration_set_event_destinations.go +++ b/resources/services/ses/configuration_set_event_destinations.go @@ -3,12 +3,13 @@ package ses import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/sesv2" "github.com/aws/aws-sdk-go-v2/service/sesv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func configurationSetEventDestinations() *schema.Table { diff --git a/resources/services/ses/configuration_sets.go b/resources/services/ses/configuration_sets.go index 79c5cbe4..dcb07867 100644 --- a/resources/services/ses/configuration_sets.go +++ b/resources/services/ses/configuration_sets.go @@ -3,12 +3,13 @@ package ses import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/sesv2" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func ConfigurationSets() *schema.Table { diff --git a/resources/services/ses/configuration_sets_mock_test.go b/resources/services/ses/configuration_sets_mock_test.go index 159d50c2..8c482f2c 100644 --- a/resources/services/ses/configuration_sets_mock_test.go +++ b/resources/services/ses/configuration_sets_mock_test.go @@ -3,13 +3,14 @@ package ses import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/sesv2" "github.com/aws/aws-sdk-go-v2/service/sesv2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildConfigurationSets(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ses/custom_verification_email_templates.go b/resources/services/ses/custom_verification_email_templates.go index e8437008..7be3a904 100644 --- a/resources/services/ses/custom_verification_email_templates.go +++ b/resources/services/ses/custom_verification_email_templates.go @@ -3,12 +3,13 @@ package ses import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/sesv2" "github.com/aws/aws-sdk-go-v2/service/sesv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func CustomVerificationEmailTemplates() *schema.Table { diff --git a/resources/services/ses/custom_verification_email_templates_mock_test.go b/resources/services/ses/custom_verification_email_templates_mock_test.go index d276f3bb..fdde5657 100644 --- a/resources/services/ses/custom_verification_email_templates_mock_test.go +++ b/resources/services/ses/custom_verification_email_templates_mock_test.go @@ -3,13 +3,14 @@ package ses import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/sesv2" "github.com/aws/aws-sdk-go-v2/service/sesv2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildCustomVerificationEmailTemplates(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ses/identities.go b/resources/services/ses/identities.go index 86fc9310..9f6a4326 100644 --- a/resources/services/ses/identities.go +++ b/resources/services/ses/identities.go @@ -3,14 +3,15 @@ package ses import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/ses/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/sesv2" "github.com/aws/aws-sdk-go-v2/service/sesv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/ses/models" ) func Identities() *schema.Table { diff --git a/resources/services/ses/identities_mock_test.go b/resources/services/ses/identities_mock_test.go index 4292d30e..993be64a 100644 --- a/resources/services/ses/identities_mock_test.go +++ b/resources/services/ses/identities_mock_test.go @@ -3,13 +3,14 @@ package ses import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/sesv2" "github.com/aws/aws-sdk-go-v2/service/sesv2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildIdentities(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ses/templates.go b/resources/services/ses/templates.go index 726e8e5e..dbc1cec7 100644 --- a/resources/services/ses/templates.go +++ b/resources/services/ses/templates.go @@ -3,13 +3,14 @@ package ses import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/ses/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/sesv2" "github.com/aws/aws-sdk-go-v2/service/sesv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/ses/models" ) func Templates() *schema.Table { diff --git a/resources/services/ses/templates_mock_test.go b/resources/services/ses/templates_mock_test.go index b814d8dd..ee2ce7a4 100644 --- a/resources/services/ses/templates_mock_test.go +++ b/resources/services/ses/templates_mock_test.go @@ -3,13 +3,14 @@ package ses import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/sesv2" "github.com/aws/aws-sdk-go-v2/service/sesv2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildSESTemplates(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/shield/protection_groups.go b/resources/services/shield/protection_groups.go index 1b7bb008..588988dd 100644 --- a/resources/services/shield/protection_groups.go +++ b/resources/services/shield/protection_groups.go @@ -3,13 +3,14 @@ package shield import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/shield" "github.com/aws/aws-sdk-go-v2/service/shield/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func ProtectionGroups() *schema.Table { @@ -57,6 +58,7 @@ func fetchShieldProtectionGroups(ctx context.Context, meta schema.ClientMeta, pa } return nil } + func resolveShieldProtectionGroupTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { r := resource.Item.(types.ProtectionGroup) cl := meta.(*client.Client) diff --git a/resources/services/shield/protection_groups_mock_test.go b/resources/services/shield/protection_groups_mock_test.go index f22eab44..d5bffe64 100644 --- a/resources/services/shield/protection_groups_mock_test.go +++ b/resources/services/shield/protection_groups_mock_test.go @@ -3,12 +3,13 @@ package shield import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/shield" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildProtectionGroups(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/shield/protections.go b/resources/services/shield/protections.go index 10d525ad..e0fbe5e1 100644 --- a/resources/services/shield/protections.go +++ b/resources/services/shield/protections.go @@ -3,13 +3,14 @@ package shield import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/shield" "github.com/aws/aws-sdk-go-v2/service/shield/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Protections() *schema.Table { diff --git a/resources/services/shield/protections_mock_test.go b/resources/services/shield/protections_mock_test.go index 70a4615a..7e1c885c 100644 --- a/resources/services/shield/protections_mock_test.go +++ b/resources/services/shield/protections_mock_test.go @@ -3,12 +3,13 @@ package shield import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/shield" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildProtections(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/shield/subscriptions.go b/resources/services/shield/subscriptions.go index 54308b0e..fb6dc8aa 100644 --- a/resources/services/shield/subscriptions.go +++ b/resources/services/shield/subscriptions.go @@ -3,12 +3,13 @@ package shield import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/shield" "github.com/aws/aws-sdk-go-v2/service/shield/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Subscriptions() *schema.Table { diff --git a/resources/services/shield/subscriptions_mock_test.go b/resources/services/shield/subscriptions_mock_test.go index 9929623d..23657891 100644 --- a/resources/services/shield/subscriptions_mock_test.go +++ b/resources/services/shield/subscriptions_mock_test.go @@ -3,12 +3,13 @@ package shield import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/shield" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildSubscriptions(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/signer/profiles.go b/resources/services/signer/profiles.go index 4102ace6..d1c0f6af 100644 --- a/resources/services/signer/profiles.go +++ b/resources/services/signer/profiles.go @@ -3,12 +3,13 @@ package signer import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/signer" "github.com/aws/aws-sdk-go-v2/service/signer/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Profiles() *schema.Table { diff --git a/resources/services/signer/profiles_mock_test.go b/resources/services/signer/profiles_mock_test.go index afda9a68..1cc70c2b 100644 --- a/resources/services/signer/profiles_mock_test.go +++ b/resources/services/signer/profiles_mock_test.go @@ -3,13 +3,14 @@ package signer import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/signer" "github.com/aws/aws-sdk-go-v2/service/signer/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildProfiles(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/sns/subscriptions.go b/resources/services/sns/subscriptions.go index c6559b33..596b4150 100644 --- a/resources/services/sns/subscriptions.go +++ b/resources/services/sns/subscriptions.go @@ -3,16 +3,17 @@ package sns import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/sns/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/sns" "github.com/aws/aws-sdk-go-v2/service/sns/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" "github.com/mitchellh/mapstructure" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/sns/models" ) func Subscriptions() *schema.Table { diff --git a/resources/services/sns/subscriptions_mock_test.go b/resources/services/sns/subscriptions_mock_test.go index 95350e13..9ee22b45 100644 --- a/resources/services/sns/subscriptions_mock_test.go +++ b/resources/services/sns/subscriptions_mock_test.go @@ -4,14 +4,14 @@ import ( "testing" "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/sns" + "github.com/aws/aws-sdk-go-v2/service/sns/types" + "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/OpsHelmInc/cloudquery/v2/client/mocks" - "github.com/aws/aws-sdk-go-v2/service/sns" - "github.com/aws/aws-sdk-go-v2/service/sns/types" - "github.com/cloudquery/plugin-sdk/v4/faker" - "github.com/golang/mock/gomock" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildSnsSubscriptions(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/sns/topics.go b/resources/services/sns/topics.go index 67f21a62..e4905a9e 100644 --- a/resources/services/sns/topics.go +++ b/resources/services/sns/topics.go @@ -3,15 +3,16 @@ package sns import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/sns/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/sns" "github.com/aws/aws-sdk-go-v2/service/sns/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" "github.com/mitchellh/mapstructure" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/sns/models" ) func Topics() *schema.Table { diff --git a/resources/services/sns/topics_mock_test.go b/resources/services/sns/topics_mock_test.go index 3cb65213..34c65769 100644 --- a/resources/services/sns/topics_mock_test.go +++ b/resources/services/sns/topics_mock_test.go @@ -3,13 +3,14 @@ package sns import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/sns" "github.com/aws/aws-sdk-go-v2/service/sns/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildSnsTopics(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/sqs/queues.go b/resources/services/sqs/queues.go index 4edebb93..107ef209 100644 --- a/resources/services/sqs/queues.go +++ b/resources/services/sqs/queues.go @@ -3,16 +3,17 @@ package sqs import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/sqs/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/aws/aws-sdk-go-v2/service/sqs/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" "github.com/mitchellh/mapstructure" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/sqs/models" ) func Queues() *schema.Table { diff --git a/resources/services/sqs/queues_mock_test.go b/resources/services/sqs/queues_mock_test.go index 617e62d9..92142a6d 100644 --- a/resources/services/sqs/queues_mock_test.go +++ b/resources/services/sqs/queues_mock_test.go @@ -3,11 +3,12 @@ package sqs import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/aws/aws-sdk-go-v2/service/sqs/types" "github.com/golang/mock/gomock" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" ) func buildSQSQueues(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ssm/associations.go b/resources/services/ssm/associations.go index 460206a0..17f1b293 100644 --- a/resources/services/ssm/associations.go +++ b/resources/services/ssm/associations.go @@ -3,12 +3,13 @@ package ssm import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/ssm" "github.com/aws/aws-sdk-go-v2/service/ssm/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Associations() *schema.Table { diff --git a/resources/services/ssm/associations_mock_test.go b/resources/services/ssm/associations_mock_test.go index 1167a4c1..a04b9a7e 100644 --- a/resources/services/ssm/associations_mock_test.go +++ b/resources/services/ssm/associations_mock_test.go @@ -3,13 +3,14 @@ package ssm import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ssm" "github.com/aws/aws-sdk-go-v2/service/ssm/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildAssociations(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ssm/compliance_summary_items.go b/resources/services/ssm/compliance_summary_items.go index ecf44718..d4762eb8 100644 --- a/resources/services/ssm/compliance_summary_items.go +++ b/resources/services/ssm/compliance_summary_items.go @@ -3,13 +3,14 @@ package ssm import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ssm" "github.com/aws/aws-sdk-go-v2/service/ssm/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func ComplianceSummaryItems() *schema.Table { diff --git a/resources/services/ssm/compliance_summary_items_mock_test.go b/resources/services/ssm/compliance_summary_items_mock_test.go index c0d04242..9be7faa2 100644 --- a/resources/services/ssm/compliance_summary_items_mock_test.go +++ b/resources/services/ssm/compliance_summary_items_mock_test.go @@ -3,13 +3,14 @@ package ssm import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ssm" "github.com/aws/aws-sdk-go-v2/service/ssm/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildComplianceSummaryItems(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ssm/document_versions.go b/resources/services/ssm/document_versions.go index 1e456a39..b5d1e9cf 100644 --- a/resources/services/ssm/document_versions.go +++ b/resources/services/ssm/document_versions.go @@ -3,13 +3,14 @@ package ssm import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ssm" "github.com/aws/aws-sdk-go-v2/service/ssm/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func documentVersions() *schema.Table { diff --git a/resources/services/ssm/documents.go b/resources/services/ssm/documents.go index f2b2ea25..2b6fb738 100644 --- a/resources/services/ssm/documents.go +++ b/resources/services/ssm/documents.go @@ -4,15 +4,16 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ssm" "github.com/aws/aws-sdk-go-v2/service/ssm/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Documents() *schema.Table { diff --git a/resources/services/ssm/documents_mock_test.go b/resources/services/ssm/documents_mock_test.go index a4b587d2..630bf0e3 100644 --- a/resources/services/ssm/documents_mock_test.go +++ b/resources/services/ssm/documents_mock_test.go @@ -3,14 +3,15 @@ package ssm import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ssm" "github.com/aws/aws-sdk-go-v2/service/ssm/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildSSMDocuments(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ssm/instance_compliance_items.go b/resources/services/ssm/instance_compliance_items.go index 45cc1af1..27504d30 100644 --- a/resources/services/ssm/instance_compliance_items.go +++ b/resources/services/ssm/instance_compliance_items.go @@ -3,12 +3,13 @@ package ssm import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/ssm" "github.com/aws/aws-sdk-go-v2/service/ssm/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func instanceComplianceItems() *schema.Table { diff --git a/resources/services/ssm/instance_patches.go b/resources/services/ssm/instance_patches.go index 7bd6cd2a..d527a569 100644 --- a/resources/services/ssm/instance_patches.go +++ b/resources/services/ssm/instance_patches.go @@ -3,12 +3,13 @@ package ssm import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/ssm" "github.com/aws/aws-sdk-go-v2/service/ssm/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func instancePatches() *schema.Table { diff --git a/resources/services/ssm/instances.go b/resources/services/ssm/instances.go index 7eb597e1..cb2018ad 100644 --- a/resources/services/ssm/instances.go +++ b/resources/services/ssm/instances.go @@ -4,14 +4,15 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ssm" "github.com/aws/aws-sdk-go-v2/service/ssm/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Instances() *schema.Table { diff --git a/resources/services/ssm/instances_mock_test.go b/resources/services/ssm/instances_mock_test.go index 59b08071..e42ec031 100644 --- a/resources/services/ssm/instances_mock_test.go +++ b/resources/services/ssm/instances_mock_test.go @@ -3,14 +3,15 @@ package ssm import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ssm" "github.com/aws/aws-sdk-go-v2/service/ssm/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildSSMInstances(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ssm/parameters.go b/resources/services/ssm/parameters.go index 24ae702a..59a97437 100644 --- a/resources/services/ssm/parameters.go +++ b/resources/services/ssm/parameters.go @@ -4,15 +4,16 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/ssm" "github.com/aws/aws-sdk-go-v2/service/ssm/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Parameters() *schema.Table { @@ -63,6 +64,7 @@ func fetchSsmParameters(ctx context.Context, meta schema.ClientMeta, parent *sch } return nil } + func resolveParameterTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { cl := meta.(*client.Client) svc := cl.Services(client.AWSServiceSsm).Ssm diff --git a/resources/services/ssm/parameters_mock_test.go b/resources/services/ssm/parameters_mock_test.go index 928dfd14..5e099d20 100644 --- a/resources/services/ssm/parameters_mock_test.go +++ b/resources/services/ssm/parameters_mock_test.go @@ -3,13 +3,14 @@ package ssm import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ssm" "github.com/aws/aws-sdk-go-v2/service/ssm/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildParameters(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ssoadmin/account_assignments.go b/resources/services/ssoadmin/account_assignments.go index 414cc3d9..e74d8653 100644 --- a/resources/services/ssoadmin/account_assignments.go +++ b/resources/services/ssoadmin/account_assignments.go @@ -3,13 +3,14 @@ package ssoadmin import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ssoadmin" "github.com/aws/aws-sdk-go-v2/service/ssoadmin/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func accountAssignments() *schema.Table { diff --git a/resources/services/ssoadmin/customer_managed_policies.go b/resources/services/ssoadmin/customer_managed_policies.go index 0632a884..f66078bd 100644 --- a/resources/services/ssoadmin/customer_managed_policies.go +++ b/resources/services/ssoadmin/customer_managed_policies.go @@ -3,12 +3,13 @@ package ssoadmin import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/ssoadmin" "github.com/aws/aws-sdk-go-v2/service/ssoadmin/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func customerManagedPolicies() *schema.Table { diff --git a/resources/services/ssoadmin/inline_policies.go b/resources/services/ssoadmin/inline_policies.go index 5f6f8203..61d053d2 100644 --- a/resources/services/ssoadmin/inline_policies.go +++ b/resources/services/ssoadmin/inline_policies.go @@ -5,13 +5,14 @@ import ( "encoding/json" "net/url" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/ssoadmin" "github.com/aws/aws-sdk-go-v2/service/ssoadmin/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func inlinePolicies() *schema.Table { diff --git a/resources/services/ssoadmin/instances.go b/resources/services/ssoadmin/instances.go index 11783118..a82667ed 100644 --- a/resources/services/ssoadmin/instances.go +++ b/resources/services/ssoadmin/instances.go @@ -3,11 +3,12 @@ package ssoadmin import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/service/ssoadmin" "github.com/aws/aws-sdk-go-v2/service/ssoadmin/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Instances() *schema.Table { diff --git a/resources/services/ssoadmin/instances_mock_test.go b/resources/services/ssoadmin/instances_mock_test.go index f0da9004..88de50af 100644 --- a/resources/services/ssoadmin/instances_mock_test.go +++ b/resources/services/ssoadmin/instances_mock_test.go @@ -3,13 +3,14 @@ package ssoadmin import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/ssoadmin" "github.com/aws/aws-sdk-go-v2/service/ssoadmin/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildInstances(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/ssoadmin/managed_policies.go b/resources/services/ssoadmin/managed_policies.go index 1319228d..c79cd3fb 100644 --- a/resources/services/ssoadmin/managed_policies.go +++ b/resources/services/ssoadmin/managed_policies.go @@ -3,12 +3,13 @@ package ssoadmin import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/ssoadmin" "github.com/aws/aws-sdk-go-v2/service/ssoadmin/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func managedPolicies() *schema.Table { diff --git a/resources/services/ssoadmin/permission_boundaries.go b/resources/services/ssoadmin/permission_boundaries.go index a7b0742d..6447a471 100644 --- a/resources/services/ssoadmin/permission_boundaries.go +++ b/resources/services/ssoadmin/permission_boundaries.go @@ -3,12 +3,13 @@ package ssoadmin import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/ssoadmin" "github.com/aws/aws-sdk-go-v2/service/ssoadmin/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func permissionsBoundaries() *schema.Table { diff --git a/resources/services/ssoadmin/permission_sets.go b/resources/services/ssoadmin/permission_sets.go index 46b52d3d..06fb7651 100644 --- a/resources/services/ssoadmin/permission_sets.go +++ b/resources/services/ssoadmin/permission_sets.go @@ -4,12 +4,12 @@ import ( "context" "github.com/apache/arrow/go/v16/arrow" - - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/aws/aws-sdk-go-v2/service/ssoadmin" "github.com/aws/aws-sdk-go-v2/service/ssoadmin/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func permissionSets() *schema.Table { diff --git a/resources/services/stepfunctions/activities.go b/resources/services/stepfunctions/activities.go index 76d0a98b..0d4bea30 100644 --- a/resources/services/stepfunctions/activities.go +++ b/resources/services/stepfunctions/activities.go @@ -3,12 +3,13 @@ package stepfunctions import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/sfn" "github.com/aws/aws-sdk-go-v2/service/sfn/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Activities() *schema.Table { diff --git a/resources/services/stepfunctions/activities_test.go b/resources/services/stepfunctions/activities_test.go index d2644db5..6ce1b5d0 100644 --- a/resources/services/stepfunctions/activities_test.go +++ b/resources/services/stepfunctions/activities_test.go @@ -3,13 +3,14 @@ package stepfunctions import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/sfn" "github.com/aws/aws-sdk-go-v2/service/sfn/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildActivities(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/stepfunctions/executions.go b/resources/services/stepfunctions/executions.go index 880ba252..642db735 100644 --- a/resources/services/stepfunctions/executions.go +++ b/resources/services/stepfunctions/executions.go @@ -3,12 +3,13 @@ package stepfunctions import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/sfn" "github.com/aws/aws-sdk-go-v2/service/sfn/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func executions() *schema.Table { diff --git a/resources/services/stepfunctions/executions_map_runs.go b/resources/services/stepfunctions/executions_map_runs.go index 1c0d29be..00d64cee 100644 --- a/resources/services/stepfunctions/executions_map_runs.go +++ b/resources/services/stepfunctions/executions_map_runs.go @@ -3,12 +3,13 @@ package stepfunctions import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/sfn" "github.com/aws/aws-sdk-go-v2/service/sfn/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func mapRuns() *schema.Table { diff --git a/resources/services/stepfunctions/executions_map_runs_executions.go b/resources/services/stepfunctions/executions_map_runs_executions.go index 2884bea8..2e75768b 100644 --- a/resources/services/stepfunctions/executions_map_runs_executions.go +++ b/resources/services/stepfunctions/executions_map_runs_executions.go @@ -3,11 +3,12 @@ package stepfunctions import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/sfn" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func mapRunExecutions() *schema.Table { diff --git a/resources/services/stepfunctions/state_machines.go b/resources/services/stepfunctions/state_machines.go index d9d99cd4..4627b1a7 100644 --- a/resources/services/stepfunctions/state_machines.go +++ b/resources/services/stepfunctions/state_machines.go @@ -3,13 +3,14 @@ package stepfunctions import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/sfn" "github.com/aws/aws-sdk-go-v2/service/sfn/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func StateMachines() *schema.Table { diff --git a/resources/services/stepfunctions/state_machines_test.go b/resources/services/stepfunctions/state_machines_test.go index 02fbae3f..3cd8d7ba 100644 --- a/resources/services/stepfunctions/state_machines_test.go +++ b/resources/services/stepfunctions/state_machines_test.go @@ -3,13 +3,14 @@ package stepfunctions import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/sfn" "github.com/aws/aws-sdk-go-v2/service/sfn/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildStateMachines(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/timestream/databases.go b/resources/services/timestream/databases.go index d168c707..6321f4a1 100644 --- a/resources/services/timestream/databases.go +++ b/resources/services/timestream/databases.go @@ -3,14 +3,15 @@ package timestream import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/timestreamwrite" "github.com/aws/aws-sdk-go-v2/service/timestreamwrite/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Databases() *schema.Table { diff --git a/resources/services/timestream/databases_mock_test.go b/resources/services/timestream/databases_mock_test.go index ee4e6d55..3c20b10c 100644 --- a/resources/services/timestream/databases_mock_test.go +++ b/resources/services/timestream/databases_mock_test.go @@ -3,13 +3,14 @@ package timestream import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/timestreamwrite" "github.com/aws/aws-sdk-go-v2/service/timestreamwrite/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildTimestreamDatabasesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/timestream/tables.go b/resources/services/timestream/tables.go index 4b0ed48d..a3069ec1 100644 --- a/resources/services/timestream/tables.go +++ b/resources/services/timestream/tables.go @@ -3,13 +3,14 @@ package timestream import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/timestreamwrite" "github.com/aws/aws-sdk-go-v2/service/timestreamwrite/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func tables() *schema.Table { diff --git a/resources/services/transfer/servers.go b/resources/services/transfer/servers.go index 3e8837e1..1bf14b0f 100644 --- a/resources/services/transfer/servers.go +++ b/resources/services/transfer/servers.go @@ -3,14 +3,15 @@ package transfer import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/transfer" "github.com/aws/aws-sdk-go-v2/service/transfer/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Servers() *schema.Table { @@ -41,6 +42,7 @@ func Servers() *schema.Table { }, } } + func fetchTransferServers(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { cl := meta.(*client.Client) svc := cl.Services(client.AWSServiceTransfer).Transfer diff --git a/resources/services/transfer/servers_test.go b/resources/services/transfer/servers_test.go index a3833720..1fdd73bb 100644 --- a/resources/services/transfer/servers_test.go +++ b/resources/services/transfer/servers_test.go @@ -3,14 +3,15 @@ package transfer import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/transfer" "github.com/aws/aws-sdk-go-v2/service/transfer/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildServersMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/waf/rule_groups.go b/resources/services/waf/rule_groups.go index 03d7be6b..f279690f 100644 --- a/resources/services/waf/rule_groups.go +++ b/resources/services/waf/rule_groups.go @@ -4,15 +4,16 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/waf" "github.com/aws/aws-sdk-go-v2/service/waf/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func RuleGroups() *schema.Table { @@ -74,6 +75,7 @@ func fetchWafRuleGroups(ctx context.Context, meta schema.ClientMeta, parent *sch } return nil } + func resolveWafRuleGroupArn(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { cl := meta.(*client.Client) ruleGroup := resource.Item.(*types.RuleGroup) @@ -85,6 +87,7 @@ func resolveWafRuleGroupArn(ctx context.Context, meta schema.ClientMeta, resourc Resource: fmt.Sprintf("rulegroup/%s", aws.ToString(ruleGroup.RuleGroupId)), }.String()) } + func resolveWafRuleGroupRuleIds(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { ruleGroup := resource.Item.(*types.RuleGroup) @@ -111,6 +114,7 @@ func resolveWafRuleGroupRuleIds(ctx context.Context, meta schema.ClientMeta, res } return resource.Set("rule_ids", ruleIDs) } + func resolveWafRuleGroupTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { ruleGroup := resource.Item.(*types.RuleGroup) diff --git a/resources/services/waf/rule_groups_mock_test.go b/resources/services/waf/rule_groups_mock_test.go index b547f858..f4ac0cd5 100644 --- a/resources/services/waf/rule_groups_mock_test.go +++ b/resources/services/waf/rule_groups_mock_test.go @@ -3,13 +3,14 @@ package waf import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/waf" "github.com/aws/aws-sdk-go-v2/service/waf/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildWAFRuleGroupsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/waf/rules.go b/resources/services/waf/rules.go index 9dfe59f7..a17a88ca 100644 --- a/resources/services/waf/rules.go +++ b/resources/services/waf/rules.go @@ -4,15 +4,16 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/waf" "github.com/aws/aws-sdk-go-v2/service/waf/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Rules() *schema.Table { diff --git a/resources/services/waf/rules_mock_test.go b/resources/services/waf/rules_mock_test.go index 1159daa0..e79fc16c 100644 --- a/resources/services/waf/rules_mock_test.go +++ b/resources/services/waf/rules_mock_test.go @@ -3,13 +3,14 @@ package waf import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/waf" "github.com/aws/aws-sdk-go-v2/service/waf/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildWAFRulesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/waf/web_acls.go b/resources/services/waf/web_acls.go index c76fa665..b100146d 100644 --- a/resources/services/waf/web_acls.go +++ b/resources/services/waf/web_acls.go @@ -3,14 +3,15 @@ package waf import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/waf" "github.com/aws/aws-sdk-go-v2/service/waf/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func WebAcls() *schema.Table { @@ -93,6 +94,7 @@ func fetchWafWebAcls(ctx context.Context, meta schema.ClientMeta, _ *schema.Reso } return nil } + func resolveWafWebACLTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { webACL := resource.Item.(*WebACLWrapper) diff --git a/resources/services/waf/web_acls_mock_test.go b/resources/services/waf/web_acls_mock_test.go index b0477260..7ae57275 100644 --- a/resources/services/waf/web_acls_mock_test.go +++ b/resources/services/waf/web_acls_mock_test.go @@ -3,13 +3,14 @@ package waf import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/waf" "github.com/aws/aws-sdk-go-v2/service/waf/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildWAFWebACLMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/wafregional/rate_based_rules.go b/resources/services/wafregional/rate_based_rules.go index a104b876..cc9e202a 100644 --- a/resources/services/wafregional/rate_based_rules.go +++ b/resources/services/wafregional/rate_based_rules.go @@ -4,15 +4,16 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/wafregional" "github.com/aws/aws-sdk-go-v2/service/wafregional/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func RateBasedRules() *schema.Table { @@ -76,9 +77,11 @@ func fetchWafregionalRateBasedRules(ctx context.Context, meta schema.ClientMeta, } return nil } + func resolveWafregionalRateBasedRuleArn(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { return resource.Set(c.Name, rateBasedRuleARN(meta, *resource.Item.(types.RateBasedRule).RuleId)) } + func resolveWafregionalRateBasedRuleTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { cl := meta.(*client.Client) svc := cl.Services(client.AWSServiceWafregional).Wafregional diff --git a/resources/services/wafregional/rate_based_rules_mock_test.go b/resources/services/wafregional/rate_based_rules_mock_test.go index 45b5b58e..c281db50 100644 --- a/resources/services/wafregional/rate_based_rules_mock_test.go +++ b/resources/services/wafregional/rate_based_rules_mock_test.go @@ -4,14 +4,15 @@ import ( "fmt" "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/wafregional" "github.com/aws/aws-sdk-go-v2/service/wafregional/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRateBasedRulesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/wafregional/rule_groups.go b/resources/services/wafregional/rule_groups.go index aaadf137..3e34e15a 100644 --- a/resources/services/wafregional/rule_groups.go +++ b/resources/services/wafregional/rule_groups.go @@ -4,15 +4,16 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/wafregional" "github.com/aws/aws-sdk-go-v2/service/wafregional/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func RuleGroups() *schema.Table { diff --git a/resources/services/wafregional/rule_groups_mock_test.go b/resources/services/wafregional/rule_groups_mock_test.go index 41ecdea1..a9a24de6 100644 --- a/resources/services/wafregional/rule_groups_mock_test.go +++ b/resources/services/wafregional/rule_groups_mock_test.go @@ -4,14 +4,15 @@ import ( "fmt" "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/wafregional" "github.com/aws/aws-sdk-go-v2/service/wafregional/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRuleGroupsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/wafregional/rules.go b/resources/services/wafregional/rules.go index 02677428..eff1bd27 100644 --- a/resources/services/wafregional/rules.go +++ b/resources/services/wafregional/rules.go @@ -4,15 +4,16 @@ import ( "context" "fmt" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/wafregional" "github.com/aws/aws-sdk-go-v2/service/wafregional/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Rules() *schema.Table { diff --git a/resources/services/wafregional/rules_mock_test.go b/resources/services/wafregional/rules_mock_test.go index 11ada752..f92f1c0f 100644 --- a/resources/services/wafregional/rules_mock_test.go +++ b/resources/services/wafregional/rules_mock_test.go @@ -4,14 +4,15 @@ import ( "fmt" "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/wafregional" "github.com/aws/aws-sdk-go-v2/service/wafregional/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRulesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/wafregional/web_acls.go b/resources/services/wafregional/web_acls.go index 994a8711..7287c4c9 100644 --- a/resources/services/wafregional/web_acls.go +++ b/resources/services/wafregional/web_acls.go @@ -3,14 +3,15 @@ package wafregional import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/wafregional" "github.com/aws/aws-sdk-go-v2/service/wafregional/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func WebAcls() *schema.Table { @@ -80,6 +81,7 @@ func fetchWafregionalWebAcls(ctx context.Context, meta schema.ClientMeta, parent } return nil } + func resolveWafregionalWebACLTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { cl := meta.(*client.Client) svc := cl.Services(client.AWSServiceWafregional).Wafregional diff --git a/resources/services/wafregional/web_acls_mock_test.go b/resources/services/wafregional/web_acls_mock_test.go index 6a80028b..2f5626ac 100644 --- a/resources/services/wafregional/web_acls_mock_test.go +++ b/resources/services/wafregional/web_acls_mock_test.go @@ -3,13 +3,14 @@ package wafregional import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/wafregional" "github.com/aws/aws-sdk-go-v2/service/wafregional/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildWebACLsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/wafv2/ipsets.go b/resources/services/wafv2/ipsets.go index 75fdbbd2..c031a635 100644 --- a/resources/services/wafv2/ipsets.go +++ b/resources/services/wafv2/ipsets.go @@ -4,14 +4,15 @@ import ( "context" "net" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/wafv2" "github.com/aws/aws-sdk-go-v2/service/wafv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Ipsets() *schema.Table { diff --git a/resources/services/wafv2/ipsets_mock_test.go b/resources/services/wafv2/ipsets_mock_test.go index dedd5c2b..206eead0 100644 --- a/resources/services/wafv2/ipsets_mock_test.go +++ b/resources/services/wafv2/ipsets_mock_test.go @@ -3,14 +3,15 @@ package wafv2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/wafv2" "github.com/aws/aws-sdk-go-v2/service/wafv2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildIpsetsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/wafv2/regex_pattern_sets.go b/resources/services/wafv2/regex_pattern_sets.go index 3831509d..f2ee83e8 100644 --- a/resources/services/wafv2/regex_pattern_sets.go +++ b/resources/services/wafv2/regex_pattern_sets.go @@ -3,14 +3,15 @@ package wafv2 import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/wafv2" "github.com/aws/aws-sdk-go-v2/service/wafv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func RegexPatternSets() *schema.Table { diff --git a/resources/services/wafv2/regex_pattern_sets_mock_test.go b/resources/services/wafv2/regex_pattern_sets_mock_test.go index a5b30fb1..17991be0 100644 --- a/resources/services/wafv2/regex_pattern_sets_mock_test.go +++ b/resources/services/wafv2/regex_pattern_sets_mock_test.go @@ -3,14 +3,15 @@ package wafv2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/wafv2" "github.com/aws/aws-sdk-go-v2/service/wafv2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildRegexPatternSetsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/wafv2/rule_groups.go b/resources/services/wafv2/rule_groups.go index 45a37956..a62763f1 100644 --- a/resources/services/wafv2/rule_groups.go +++ b/resources/services/wafv2/rule_groups.go @@ -5,14 +5,15 @@ import ( "encoding/json" "errors" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/wafv2" "github.com/aws/aws-sdk-go-v2/service/wafv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func RuleGroups() *schema.Table { @@ -118,6 +119,7 @@ func resolveRuleGroupTags(ctx context.Context, meta schema.ClientMeta, resource } return resource.Set(c.Name, outputTags) } + func resolveWafv2ruleGroupPolicy(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { ruleGroup := resource.Item.(*types.RuleGroup) diff --git a/resources/services/wafv2/rule_groups_mock_test.go b/resources/services/wafv2/rule_groups_mock_test.go index ddd11b4c..2748895f 100644 --- a/resources/services/wafv2/rule_groups_mock_test.go +++ b/resources/services/wafv2/rule_groups_mock_test.go @@ -3,14 +3,15 @@ package wafv2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/wafv2" "github.com/aws/aws-sdk-go-v2/service/wafv2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildWAFV2RuleGroupsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/wafv2/web_acls.go b/resources/services/wafv2/web_acls.go index 411afe2d..c23013be 100644 --- a/resources/services/wafv2/web_acls.go +++ b/resources/services/wafv2/web_acls.go @@ -4,16 +4,17 @@ import ( "context" "errors" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/resources/services/wafv2/models" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudfront" "github.com/aws/aws-sdk-go-v2/service/wafv2" "github.com/aws/aws-sdk-go-v2/service/wafv2/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" + "github.com/OpsHelmInc/cloudquery/v2/resources/services/wafv2/models" ) func WebAcls() *schema.Table { @@ -167,6 +168,7 @@ func resolveWafv2webACLResourcesForWebACL(ctx context.Context, meta schema.Clien } return errs } + func resolveWebACLTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { webACL := resource.Item.(*models.WebACLWrapper) diff --git a/resources/services/wafv2/web_acls_mock_test.go b/resources/services/wafv2/web_acls_mock_test.go index 11829ac8..88a9cd70 100644 --- a/resources/services/wafv2/web_acls_mock_test.go +++ b/resources/services/wafv2/web_acls_mock_test.go @@ -3,16 +3,17 @@ package wafv2 import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudfront" cftypes "github.com/aws/aws-sdk-go-v2/service/cloudfront/types" "github.com/aws/aws-sdk-go-v2/service/wafv2" "github.com/aws/aws-sdk-go-v2/service/wafv2/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildWAFV2WebACLMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/wellarchitected/lens_review_improvements.go b/resources/services/wellarchitected/lens_review_improvements.go index 49d956ea..ca0b98cb 100644 --- a/resources/services/wellarchitected/lens_review_improvements.go +++ b/resources/services/wellarchitected/lens_review_improvements.go @@ -3,13 +3,14 @@ package wellarchitected import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/wellarchitected" "github.com/aws/aws-sdk-go-v2/service/wellarchitected/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func lensReviewImprovements() *schema.Table { diff --git a/resources/services/wellarchitected/lens_review_improvements_mock_test.go b/resources/services/wellarchitected/lens_review_improvements_mock_test.go index 20e4288f..84aa90ed 100644 --- a/resources/services/wellarchitected/lens_review_improvements_mock_test.go +++ b/resources/services/wellarchitected/lens_review_improvements_mock_test.go @@ -3,13 +3,14 @@ package wellarchitected import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/wellarchitected" "github.com/aws/aws-sdk-go-v2/service/wellarchitected/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildLensReviewImprovements(t *testing.T, m *mocks.MockWellarchitectedClient, diff --git a/resources/services/wellarchitected/lens_reviews.go b/resources/services/wellarchitected/lens_reviews.go index cd68dd8d..1462ee39 100644 --- a/resources/services/wellarchitected/lens_reviews.go +++ b/resources/services/wellarchitected/lens_reviews.go @@ -3,13 +3,14 @@ package wellarchitected import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/wellarchitected" "github.com/aws/aws-sdk-go-v2/service/wellarchitected/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func lensReviews() *schema.Table { diff --git a/resources/services/wellarchitected/lens_reviews_mock_test.go b/resources/services/wellarchitected/lens_reviews_mock_test.go index 7a984af6..ed6efd4b 100644 --- a/resources/services/wellarchitected/lens_reviews_mock_test.go +++ b/resources/services/wellarchitected/lens_reviews_mock_test.go @@ -3,13 +3,14 @@ package wellarchitected import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/wellarchitected" "github.com/aws/aws-sdk-go-v2/service/wellarchitected/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildLensReviews(t *testing.T, m *mocks.MockWellarchitectedClient, diff --git a/resources/services/wellarchitected/lenses.go b/resources/services/wellarchitected/lenses.go index 03219ee3..402eeff9 100644 --- a/resources/services/wellarchitected/lenses.go +++ b/resources/services/wellarchitected/lenses.go @@ -3,13 +3,14 @@ package wellarchitected import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/wellarchitected" "github.com/aws/aws-sdk-go-v2/service/wellarchitected/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) // lens info consists of types.LensSummary & types.Lens fields diff --git a/resources/services/wellarchitected/lenses_mock_test.go b/resources/services/wellarchitected/lenses_mock_test.go index b677795b..c477e9e1 100644 --- a/resources/services/wellarchitected/lenses_mock_test.go +++ b/resources/services/wellarchitected/lenses_mock_test.go @@ -3,14 +3,15 @@ package wellarchitected import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/wellarchitected" "github.com/aws/aws-sdk-go-v2/service/wellarchitected/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildLensesMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/wellarchitected/workload_milestones.go b/resources/services/wellarchitected/workload_milestones.go index dc9cdb48..cc5514ba 100644 --- a/resources/services/wellarchitected/workload_milestones.go +++ b/resources/services/wellarchitected/workload_milestones.go @@ -3,13 +3,14 @@ package wellarchitected import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/wellarchitected" "github.com/aws/aws-sdk-go-v2/service/wellarchitected/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func workloadMilestones() *schema.Table { diff --git a/resources/services/wellarchitected/workload_milestones_mock_test.go b/resources/services/wellarchitected/workload_milestones_mock_test.go index 2e14e2be..724a2df0 100644 --- a/resources/services/wellarchitected/workload_milestones_mock_test.go +++ b/resources/services/wellarchitected/workload_milestones_mock_test.go @@ -3,13 +3,14 @@ package wellarchitected import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/wellarchitected" "github.com/aws/aws-sdk-go-v2/service/wellarchitected/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildWorkloadMilestones(t *testing.T, m *mocks.MockWellarchitectedClient, workload *types.Workload) { diff --git a/resources/services/wellarchitected/workload_shares.go b/resources/services/wellarchitected/workload_shares.go index fc8c6ad7..48d3dff9 100644 --- a/resources/services/wellarchitected/workload_shares.go +++ b/resources/services/wellarchitected/workload_shares.go @@ -3,13 +3,14 @@ package wellarchitected import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/wellarchitected" "github.com/aws/aws-sdk-go-v2/service/wellarchitected/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func workloadShares() *schema.Table { diff --git a/resources/services/wellarchitected/workload_shares_mock_test.go b/resources/services/wellarchitected/workload_shares_mock_test.go index fa94b815..eb77d8f2 100644 --- a/resources/services/wellarchitected/workload_shares_mock_test.go +++ b/resources/services/wellarchitected/workload_shares_mock_test.go @@ -3,13 +3,14 @@ package wellarchitected import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/wellarchitected" "github.com/aws/aws-sdk-go-v2/service/wellarchitected/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildWorkloadShares(t *testing.T, m *mocks.MockWellarchitectedClient, workload *types.Workload) { diff --git a/resources/services/wellarchitected/workloads.go b/resources/services/wellarchitected/workloads.go index 200a42cd..9aa9b645 100644 --- a/resources/services/wellarchitected/workloads.go +++ b/resources/services/wellarchitected/workloads.go @@ -3,13 +3,14 @@ package wellarchitected import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/wellarchitected" "github.com/aws/aws-sdk-go-v2/service/wellarchitected/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Workloads() *schema.Table { diff --git a/resources/services/wellarchitected/workloads_mock_test.go b/resources/services/wellarchitected/workloads_mock_test.go index 718cd6f0..455ce941 100644 --- a/resources/services/wellarchitected/workloads_mock_test.go +++ b/resources/services/wellarchitected/workloads_mock_test.go @@ -3,14 +3,15 @@ package wellarchitected import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/wellarchitected" "github.com/aws/aws-sdk-go-v2/service/wellarchitected/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildWorkloadsMock(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/workspaces/directories.go b/resources/services/workspaces/directories.go index d4dfcf1e..17d11fff 100644 --- a/resources/services/workspaces/directories.go +++ b/resources/services/workspaces/directories.go @@ -3,13 +3,14 @@ package workspaces import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/workspaces" "github.com/aws/aws-sdk-go-v2/service/workspaces/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Directories() *schema.Table { diff --git a/resources/services/workspaces/directories_mock_test.go b/resources/services/workspaces/directories_mock_test.go index 6fb5deb9..f30f1191 100644 --- a/resources/services/workspaces/directories_mock_test.go +++ b/resources/services/workspaces/directories_mock_test.go @@ -3,13 +3,14 @@ package workspaces import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/workspaces" "github.com/aws/aws-sdk-go-v2/service/workspaces/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildDirectories(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/workspaces/workspaces.go b/resources/services/workspaces/workspaces.go index 9797c569..d87a4a67 100644 --- a/resources/services/workspaces/workspaces.go +++ b/resources/services/workspaces/workspaces.go @@ -3,13 +3,14 @@ package workspaces import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/workspaces" "github.com/aws/aws-sdk-go-v2/service/workspaces/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" ) func Workspaces() *schema.Table { diff --git a/resources/services/workspaces/workspaces_mock_test.go b/resources/services/workspaces/workspaces_mock_test.go index ee1b5baf..7d4c4cb7 100644 --- a/resources/services/workspaces/workspaces_mock_test.go +++ b/resources/services/workspaces/workspaces_mock_test.go @@ -3,13 +3,14 @@ package workspaces import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/workspaces" "github.com/aws/aws-sdk-go-v2/service/workspaces/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildWorkspaces(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/xray/groups.go b/resources/services/xray/groups.go index fb3bd1c0..63cf68af 100644 --- a/resources/services/xray/groups.go +++ b/resources/services/xray/groups.go @@ -3,13 +3,14 @@ package xray import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/xray" "github.com/aws/aws-sdk-go-v2/service/xray/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func Groups() *schema.Table { @@ -53,6 +54,7 @@ func fetchXrayGroups(ctx context.Context, meta schema.ClientMeta, parent *schema } return nil } + func resolveXrayGroupTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { group := resource.Item.(types.GroupSummary) cl := meta.(*client.Client) diff --git a/resources/services/xray/groups_mock_test.go b/resources/services/xray/groups_mock_test.go index 3a92b1c1..ac037ad3 100644 --- a/resources/services/xray/groups_mock_test.go +++ b/resources/services/xray/groups_mock_test.go @@ -3,13 +3,14 @@ package xray import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/xray" "github.com/aws/aws-sdk-go-v2/service/xray/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildGroups(t *testing.T, ctrl *gomock.Controller) client.Services { diff --git a/resources/services/xray/sampling_rules.go b/resources/services/xray/sampling_rules.go index 51cf5c9e..a2ba27aa 100644 --- a/resources/services/xray/sampling_rules.go +++ b/resources/services/xray/sampling_rules.go @@ -3,13 +3,14 @@ package xray import ( "context" - "github.com/OpsHelmInc/cloudquery/v2/client" "github.com/apache/arrow/go/v16/arrow" "github.com/aws/aws-sdk-go-v2/service/xray" "github.com/aws/aws-sdk-go-v2/service/xray/types" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/schema" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/transformers" + sdkTypes "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/types" ) func SamplingRules() *schema.Table { @@ -53,6 +54,7 @@ func fetchXraySamplingRules(ctx context.Context, meta schema.ClientMeta, parent } return nil } + func resolveXraySamplingRuleTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { sr := resource.Item.(types.SamplingRuleRecord) cl := meta.(*client.Client) diff --git a/resources/services/xray/sampling_rules_mock_test.go b/resources/services/xray/sampling_rules_mock_test.go index dfde09c7..c020ae07 100644 --- a/resources/services/xray/sampling_rules_mock_test.go +++ b/resources/services/xray/sampling_rules_mock_test.go @@ -3,13 +3,14 @@ package xray import ( "testing" - "github.com/OpsHelmInc/cloudquery/v2/client" - "github.com/OpsHelmInc/cloudquery/v2/client/mocks" "github.com/aws/aws-sdk-go-v2/service/xray" "github.com/aws/aws-sdk-go-v2/service/xray/types" - "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + + "github.com/OpsHelmInc/cloudquery/v2/client" + "github.com/OpsHelmInc/cloudquery/v2/client/mocks" + "github.com/OpsHelmInc/cloudquery/v2/plugin-sdk/faker" ) func buildSamplingRules(t *testing.T, ctrl *gomock.Controller) client.Services {