Merge branch 'master' into sfiguere_uhg-add-user-location-to-profile

.github/workflows/airflow-plugin.yml

@@ -34,29 +34,21 @@ jobs:
         include:
           # Note: this should be kept in sync with tox.ini.
           - python-version: "3.8"
-            extra_pip_requirements: "apache-airflow~=2.1.4"
-            extra_pip_extras: plugin-v1
-          - python-version: "3.8"
-            extra_pip_requirements: "apache-airflow~=2.2.4"
-            extra_pip_extras: plugin-v1
+            extra_pip_requirements: "apache-airflow~=2.3.4"
+            extra_pip_extras: test-airflow23
           - python-version: "3.10"
             extra_pip_requirements: "apache-airflow~=2.4.3"
-            extra_pip_extras: plugin-v2,test-airflow24
+            extra_pip_extras: test-airflow24
           - python-version: "3.10"
             extra_pip_requirements: "apache-airflow~=2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt"
-            extra_pip_extras: plugin-v2
           - python-version: "3.10"
             extra_pip_requirements: "apache-airflow~=2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt"
-            extra_pip_extras: plugin-v2
           - python-version: "3.10"
             extra_pip_requirements: "apache-airflow~=2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt"
-            extra_pip_extras: plugin-v2
           - python-version: "3.11"
             extra_pip_requirements: "apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt"
-            extra_pip_extras: plugin-v2
           - python-version: "3.11"
-            extra_pip_requirements: "apache-airflow~=2.10.2 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.2/constraints-3.11.txt"
-            extra_pip_extras: plugin-v2
+            extra_pip_requirements: "apache-airflow~=2.10.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.3/constraints-3.11.txt"
       fail-fast: false
     steps:
       - name: Set up JDK 17
@@ -88,10 +80,10 @@ jobs:
             !**/binary/**
       - name: Upload coverage to Codecov
         if: always()
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: .
+          directory: ./build/coverage-reports/
           fail_ci_if_error: false
           flags: airflow,airflow-${{ matrix.extra_pip_extras }}
           name: pytest-airflow-${{ matrix.python-version }}-${{ matrix.extra_pip_requirements }}

.github/workflows/build-and-test.yml

@@ -126,6 +126,16 @@ jobs:
             !**/binary/**
       - name: Ensure codegen is updated
         uses: ./.github/actions/ensure-codegen-updated
+      - name: Upload coverage to Codecov
+        if: always()
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          directory: ./build/coverage-reports/
+          fail_ci_if_error: false
+          flags: ${{ matrix.timezone }}
+          name: ${{ matrix.command }}
+          verbose: true
 
   quickstart-compose-validation:
     runs-on: ubuntu-latest

.github/workflows/dagster-plugin.yml

@@ -66,10 +66,10 @@ jobs:
             **/junit.*.xml
       - name: Upload coverage to Codecov
         if: always()
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: .
+          directory: ./build/coverage-reports/
           fail_ci_if_error: false
           flags: dagster-${{ matrix.python-version }}-${{ matrix.extraPythonRequirement }}
           name: pytest-dagster

.github/workflows/gx-plugin.yml

@@ -70,10 +70,10 @@ jobs:
             **/junit.*.xml
       - name: Upload coverage to Codecov
         if: always()
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: .
+          directory: ./build/coverage-reports/
           fail_ci_if_error: false
           flags: gx-${{ matrix.python-version }}-${{ matrix.extraPythonRequirement }}
           name: pytest-gx

.github/workflows/metadata-ingestion.yml

@@ -94,10 +94,10 @@ jobs:
             !**/binary/**
       - name: Upload coverage to Codecov
         if: ${{ always() && matrix.python-version == '3.10' }}
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: .
+          directory: ./build/coverage-reports/
           fail_ci_if_error: false
           flags: pytest-${{ matrix.command }}
           name: pytest-${{ matrix.command }}

.github/workflows/metadata-io.yml

@@ -81,6 +81,15 @@ jobs:
             !**/binary/**
       - name: Ensure codegen is updated
         uses: ./.github/actions/ensure-codegen-updated
+      - name: Upload coverage to Codecov
+        if: ${{ always()}}
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          directory: ./build/coverage-reports/
+          fail_ci_if_error: false
+          name: metadata-io-test
+          verbose: true
 
   event-file:
     runs-on: ubuntu-latest

.github/workflows/prefect-plugin.yml

@@ -67,10 +67,10 @@ jobs:
             !**/binary/**
       - name: Upload coverage to Codecov
         if: always()
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: .
+          directory: ./build/coverage-reports/
           fail_ci_if_error: false
           flags: prefect,prefect-${{ matrix.extra_pip_extras }}
           name: pytest-prefect-${{ matrix.python-version }}

datahub-frontend/build.gradle

@@ -4,8 +4,9 @@ plugins {
   id 'org.gradle.playframework'
 }
 
-apply from: "../gradle/versioning/versioning.gradle"
+apply from: '../gradle/versioning/versioning.gradle'
 apply from: './play.gradle'
+apply from: '../gradle/coverage/java-coverage.gradle'
 
 ext {
   docker_repo = 'datahub-frontend-react'
@@ -18,6 +19,13 @@ java {
   }
 }
 
+test {
+  jacoco {
+    // jacoco instrumentation is failing when dealing with code of this dependency, excluding it.
+    excludes = ["com/gargoylesoftware/**"]
+  }
+}
+
 model {
   // Must specify the dependency here as "stage" is added by rule based model.
   tasks.myTar {

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolver.java

@@ -1,7 +1,8 @@
 package com.linkedin.datahub.graphql.resolvers.structuredproperties;
 
 import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument;
-import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_ENTITY_NAME;
+import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.buildMetadataChangeProposalWithUrn;
+import static com.linkedin.metadata.Constants.*;
 
 import com.linkedin.common.urn.Urn;
 import com.linkedin.data.template.SetMode;
@@ -12,20 +13,24 @@
 import com.linkedin.datahub.graphql.exception.AuthorizationException;
 import com.linkedin.datahub.graphql.generated.CreateStructuredPropertyInput;
 import com.linkedin.datahub.graphql.generated.StructuredPropertyEntity;
+import com.linkedin.datahub.graphql.generated.StructuredPropertySettingsInput;
 import com.linkedin.datahub.graphql.types.structuredproperty.StructuredPropertyMapper;
 import com.linkedin.entity.EntityResponse;
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.metadata.aspect.patch.builder.StructuredPropertyDefinitionPatchBuilder;
+import com.linkedin.metadata.models.StructuredPropertyUtils;
 import com.linkedin.metadata.utils.EntityKeyUtils;
 import com.linkedin.mxe.MetadataChangeProposal;
 import com.linkedin.structured.PrimitivePropertyValue;
 import com.linkedin.structured.PropertyCardinality;
 import com.linkedin.structured.PropertyValue;
 import com.linkedin.structured.StructuredPropertyKey;
+import com.linkedin.structured.StructuredPropertySettings;
 import graphql.schema.DataFetcher;
 import graphql.schema.DataFetchingEnvironment;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Objects;
-import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import javax.annotation.Nonnull;
 
@@ -54,40 +59,28 @@ public CompletableFuture<StructuredPropertyEntity> get(final DataFetchingEnviron
                   "Unable to create structured property. Please contact your admin.");
             }
             final StructuredPropertyKey key = new StructuredPropertyKey();
-            final String id = input.getId() != null ? input.getId() : UUID.randomUUID().toString();
+            final String id =
+                StructuredPropertyUtils.getPropertyId(input.getId(), input.getQualifiedName());
             key.setId(id);
             final Urn propertyUrn =
                 EntityKeyUtils.convertEntityKeyToUrn(key, STRUCTURED_PROPERTY_ENTITY_NAME);
-            StructuredPropertyDefinitionPatchBuilder builder =
-                new StructuredPropertyDefinitionPatchBuilder().urn(propertyUrn);
-
-            builder.setQualifiedName(input.getQualifiedName());
-            builder.setValueType(input.getValueType());
-            input.getEntityTypes().forEach(builder::addEntityType);
-            if (input.getDisplayName() != null) {
-              builder.setDisplayName(input.getDisplayName());
-            }
-            if (input.getDescription() != null) {
-              builder.setDescription(input.getDescription());
-            }
-            if (input.getImmutable() != null) {
-              builder.setImmutable(input.getImmutable());
-            }
-            if (input.getTypeQualifier() != null) {
-              buildTypeQualifier(input, builder);
-            }
-            if (input.getAllowedValues() != null) {
-              buildAllowedValues(input, builder);
+
+            if (_entityClient.exists(context.getOperationContext(), propertyUrn)) {
+              throw new IllegalArgumentException(
+                  "A structured property already exists with this urn");
             }
-            if (input.getCardinality() != null) {
-              builder.setCardinality(
-                  PropertyCardinality.valueOf(input.getCardinality().toString()));
+
+            List<MetadataChangeProposal> mcps = new ArrayList<>();
+
+            // first, create the property definition itself
+            mcps.add(createPropertyDefinition(context, propertyUrn, id, input));
+
+            // then add the settings aspect if we're adding any settings inputs
+            if (input.getSettings() != null) {
+              mcps.add(createPropertySettings(context, propertyUrn, input.getSettings()));
             }
-            builder.setCreated(context.getOperationContext().getAuditStamp());
-            builder.setLastModified(context.getOperationContext().getAuditStamp());
 
-            MetadataChangeProposal mcp = builder.build();
-            _entityClient.ingestProposal(context.getOperationContext(), mcp, false);
+            _entityClient.batchIngestProposals(context.getOperationContext(), mcps, false);
 
             EntityResponse response =
                 _entityClient.getV2(
@@ -103,6 +96,72 @@ public CompletableFuture<StructuredPropertyEntity> get(final DataFetchingEnviron
         });
   }
 
+  private MetadataChangeProposal createPropertySettings(
+      @Nonnull final QueryContext context,
+      @Nonnull final Urn propertyUrn,
+      final StructuredPropertySettingsInput settingsInput)
+      throws Exception {
+    StructuredPropertySettings settings = new StructuredPropertySettings();
+
+    if (settingsInput.getIsHidden() != null) {
+      settings.setIsHidden(settingsInput.getIsHidden());
+    }
+    if (settingsInput.getShowInSearchFilters() != null) {
+      settings.setShowInSearchFilters(settingsInput.getShowInSearchFilters());
+    }
+    if (settingsInput.getShowInAssetSummary() != null) {
+      settings.setShowInAssetSummary(settingsInput.getShowInAssetSummary());
+    }
+    if (settingsInput.getShowAsAssetBadge() != null) {
+      settings.setShowAsAssetBadge(settingsInput.getShowAsAssetBadge());
+    }
+    if (settingsInput.getShowInColumnsTable() != null) {
+      settings.setShowInColumnsTable(settingsInput.getShowInColumnsTable());
+    }
+    settings.setLastModified(context.getOperationContext().getAuditStamp());
+
+    StructuredPropertyUtils.validatePropertySettings(settings, true);
+
+    return buildMetadataChangeProposalWithUrn(
+        propertyUrn, STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME, settings);
+  }
+
+  private MetadataChangeProposal createPropertyDefinition(
+      @Nonnull final QueryContext context,
+      @Nonnull final Urn propertyUrn,
+      @Nonnull final String id,
+      final CreateStructuredPropertyInput input)
+      throws Exception {
+    StructuredPropertyDefinitionPatchBuilder builder =
+        new StructuredPropertyDefinitionPatchBuilder().urn(propertyUrn);
+
+    builder.setQualifiedName(id);
+    builder.setValueType(input.getValueType());
+    input.getEntityTypes().forEach(builder::addEntityType);
+    if (input.getDisplayName() != null) {
+      builder.setDisplayName(input.getDisplayName());
+    }
+    if (input.getDescription() != null) {
+      builder.setDescription(input.getDescription());
+    }
+    if (input.getImmutable() != null) {
+      builder.setImmutable(input.getImmutable());
+    }
+    if (input.getTypeQualifier() != null) {
+      buildTypeQualifier(input, builder);
+    }
+    if (input.getAllowedValues() != null) {
+      buildAllowedValues(input, builder);
+    }
+    if (input.getCardinality() != null) {
+      builder.setCardinality(PropertyCardinality.valueOf(input.getCardinality().toString()));
+    }
+    builder.setCreated(context.getOperationContext().getAuditStamp());
+    builder.setLastModified(context.getOperationContext().getAuditStamp());
+
+    return builder.build();
+  }
+
   private void buildTypeQualifier(
       @Nonnull final CreateStructuredPropertyInput input,
       @Nonnull final StructuredPropertyDefinitionPatchBuilder builder) {

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/DeleteStructuredPropertyResolver.java

@@ -6,6 +6,7 @@
 import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.authorization.AuthorizationUtils;
+import com.linkedin.datahub.graphql.concurrency.GraphQLConcurrencyUtils;
 import com.linkedin.datahub.graphql.exception.AuthorizationException;
 import com.linkedin.datahub.graphql.generated.DeleteStructuredPropertyInput;
 import com.linkedin.entity.client.EntityClient;
@@ -42,6 +43,23 @@ public CompletableFuture<Boolean> get(final DataFetchingEnvironment environment)
                   "Unable to delete structured property. Please contact your admin.");
             }
             _entityClient.deleteEntity(context.getOperationContext(), propertyUrn);
+            // Asynchronously Delete all references to the entity (to return quickly)
+            GraphQLConcurrencyUtils.supplyAsync(
+                () -> {
+                  try {
+                    _entityClient.deleteEntityReferences(
+                        context.getOperationContext(), propertyUrn);
+                  } catch (Exception e) {
+                    log.error(
+                        String.format(
+                            "Caught exception while attempting to clear all entity references for Structured Property with urn %s",
+                            propertyUrn),
+                        e);
+                  }
+                  return null;
+                },
+                this.getClass().getSimpleName(),
+                "get");
             return true;
           } catch (Exception e) {
             throw new RuntimeException(

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/RemoveStructuredPropertiesResolver.java

@@ -93,7 +93,7 @@ public CompletableFuture<com.linkedin.datahub.graphql.generated.StructuredProper
                         .getValue()
                         .data());
 
-            return StructuredPropertiesMapper.map(context, structuredProperties);
+            return StructuredPropertiesMapper.map(context, structuredProperties, assetUrn);
           } catch (Exception e) {
             throw new RuntimeException(
                 String.format("Failed to perform update against input %s", input), e);