Skip to content

Commit

Permalink
Merge branch 'master' into node-options-consilidation
Browse files Browse the repository at this point in the history
  • Loading branch information
david-leifker authored Dec 30, 2024
2 parents 5412c21 + abb6443 commit 5a7bb17
Show file tree
Hide file tree
Showing 147 changed files with 5,980 additions and 1,558 deletions.
6 changes: 3 additions & 3 deletions .github/workflows/dagster-plugin.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,11 @@ jobs:
DATAHUB_TELEMETRY_ENABLED: false
strategy:
matrix:
python-version: ["3.9", "3.10"]
python-version: ["3.9", "3.11"]
include:
- python-version: "3.9"
extraPythonRequirement: "dagster>=1.3.3"
- python-version: "3.10"
- python-version: "3.11"
extraPythonRequirement: "dagster>=1.3.3"
fail-fast: false
steps:
Expand All @@ -57,7 +57,7 @@ jobs:
if: always()
run: source metadata-ingestion-modules/dagster-plugin/venv/bin/activate && uv pip freeze
- uses: actions/upload-artifact@v4
if: ${{ always() && matrix.python-version == '3.10' && matrix.extraPythonRequirement == 'dagster>=1.3.3' }}
if: ${{ always() && matrix.python-version == '3.11' && matrix.extraPythonRequirement == 'dagster>=1.3.3' }}
with:
name: Test Results (dagster Plugin ${{ matrix.python-version}})
path: |
Expand Down
50 changes: 36 additions & 14 deletions .github/workflows/docker-unified.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1009,18 +1009,39 @@ jobs:
needs: setup
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
cypress_batch_count: ${{ steps.set-batch-count.outputs.cypress_batch_count }}
python_batch_count: ${{ steps.set-batch-count.outputs.python_batch_count }}
steps:
- id: set-batch-count
# Tests are split simply to ensure the configured number of batches for parallelization. This may need some
# increase as a new tests added increase the duration where an additional parallel batch helps.
# python_batch_count is used to split pytests in the smoke-test (batches of actual test functions)
# cypress_batch_count is used to split the collection of cypress test specs into batches.
run: |
echo "cypress_batch_count=11" >> "$GITHUB_OUTPUT"
echo "python_batch_count=5" >> "$GITHUB_OUTPUT"
- id: set-matrix
# For m batches for python and n batches for cypress, we need a test matrix of python x m + cypress x n.
# while the github action matrix generation can handle these two parts individually, there isnt a way to use the
# two generated matrices for the same job. So, produce that matrix with scripting and use the include directive
# to add it to the test matrix.
run: |
if [ '${{ needs.setup.outputs.frontend_only }}' == 'true' ]; then
echo 'matrix=["cypress_suite1","cypress_rest"]' >> "$GITHUB_OUTPUT"
elif [ '${{ needs.setup.outputs.ingestion_only }}' == 'true' ]; then
echo 'matrix=["no_cypress_suite0","no_cypress_suite1"]' >> "$GITHUB_OUTPUT"
elif [[ '${{ needs.setup.outputs.backend_change }}' == 'true' || '${{ needs.setup.outputs.smoke_test_change }}' == 'true' ]]; then
echo 'matrix=["no_cypress_suite0","no_cypress_suite1","cypress_suite1","cypress_rest"]' >> "$GITHUB_OUTPUT"
else
echo 'matrix=[]' >> "$GITHUB_OUTPUT"
python_batch_count=${{ steps.set-batch-count.outputs.python_batch_count }}
python_matrix=$(printf "{\"test_strategy\":\"pytests\",\"batch\":\"0\",\"batch_count\":\"$python_batch_count\"}"; for ((i=1;i<python_batch_count;i++)); do printf ",{\"test_strategy\":\"pytests\", \"batch_count\":\"$python_batch_count\",\"batch\":\"%d\"}" $i; done)
cypress_batch_count=${{ steps.set-batch-count.outputs.cypress_batch_count }}
cypress_matrix=$(printf "{\"test_strategy\":\"cypress\",\"batch\":\"0\",\"batch_count\":\"$cypress_batch_count\"}"; for ((i=1;i<cypress_batch_count;i++)); do printf ",{\"test_strategy\":\"cypress\", \"batch_count\":\"$cypress_batch_count\",\"batch\":\"%d\"}" $i; done)
includes=''
if [[ "${{ needs.setup.outputs.frontend_only }}" == 'true' ]]; then
includes=$cypress_matrix
elif [ "${{ needs.setup.outputs.ingestion_only }}" == 'true' ]; then
includes=$python_matrix
elif [[ "${{ needs.setup.outputs.backend_change }}" == 'true' || "${{ needs.setup.outputs.smoke_test_change }}" == 'true' ]]; then
includes="$python_matrix,$cypress_matrix"
fi
echo "matrix={\"include\":[$includes] }" >> "$GITHUB_OUTPUT"
smoke_test:
name: Run Smoke Tests
Expand All @@ -1041,8 +1062,7 @@ jobs:
]
strategy:
fail-fast: false
matrix:
test_strategy: ${{ fromJson(needs.smoke_test_matrix.outputs.matrix) }}
matrix: ${{ fromJson(needs.smoke_test_matrix.outputs.matrix) }}
if: ${{ always() && !failure() && !cancelled() && needs.smoke_test_matrix.outputs.matrix != '[]' }}
steps:
- name: Free up disk space
Expand Down Expand Up @@ -1218,6 +1238,8 @@ jobs:
CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
CLEANUP_DATA: "false"
TEST_STRATEGY: ${{ matrix.test_strategy }}
BATCH_COUNT: ${{ matrix.batch_count }}
BATCH_NUMBER: ${{ matrix.batch }}
run: |
echo "$DATAHUB_VERSION"
./gradlew --stop
Expand All @@ -1228,25 +1250,25 @@ jobs:
if: failure()
run: |
docker ps -a
TEST_STRATEGY="-${{ matrix.test_strategy }}"
TEST_STRATEGY="-${{ matrix.test_strategy }}-${{ matrix.batch }}"
source .github/scripts/docker_logs.sh
- name: Upload logs
uses: actions/upload-artifact@v3
if: failure()
with:
name: docker-logs-${{ matrix.test_strategy }}
name: docker-logs-${{ matrix.test_strategy }}-${{ matrix.batch }}
path: "docker_logs/*.log"
retention-days: 5
- name: Upload screenshots
uses: actions/upload-artifact@v3
if: failure()
with:
name: cypress-snapshots-${{ matrix.test_strategy }}
name: cypress-snapshots-${{ matrix.test_strategy }}-${{ matrix.batch }}
path: smoke-test/tests/cypress/cypress/screenshots/
- uses: actions/upload-artifact@v3
if: always()
with:
name: Test Results (smoke tests) ${{ matrix.test_strategy }}
name: Test Results (smoke tests) ${{ matrix.test_strategy }} ${{ matrix.batch }}
path: |
**/build/reports/tests/test/**
**/build/test-results/test/**
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/metadata-ingestion.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ jobs:
# DATAHUB_LOOKML_GIT_TEST_SSH_KEY: ${{ secrets.DATAHUB_LOOKML_GIT_TEST_SSH_KEY }}
strategy:
matrix:
python-version: ["3.8", "3.10"]
python-version: ["3.8", "3.11"]
command:
[
"testQuick",
Expand All @@ -43,7 +43,7 @@ jobs:
]
include:
- python-version: "3.8"
- python-version: "3.10"
- python-version: "3.11"
fail-fast: false
steps:
- name: Free up disk space
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/prefect-plugin.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ jobs:
DATAHUB_TELEMETRY_ENABLED: false
strategy:
matrix:
python-version: ["3.8", "3.9", "3.10"]
python-version: ["3.8", "3.9", "3.10", "3.11"]
fail-fast: false
steps:
- name: Set up JDK 17
Expand All @@ -52,7 +52,7 @@ jobs:
if: always()
run: source metadata-ingestion-modules/prefect-plugin/venv/bin/activate && uv pip freeze
- uses: actions/upload-artifact@v4
if: ${{ always() && matrix.python-version == '3.10'}}
if: ${{ always() && matrix.python-version == '3.11'}}
with:
name: Test Results (Prefect Plugin ${{ matrix.python-version}})
path: |
Expand Down
23 changes: 0 additions & 23 deletions .github/workflows/qodana-scan.yml

This file was deleted.

2 changes: 1 addition & 1 deletion build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ buildscript {
ext.pegasusVersion = '29.57.0'
ext.mavenVersion = '3.6.3'
ext.versionGradle = '8.11.1'
ext.springVersion = '6.1.13'
ext.springVersion = '6.1.14'
ext.springBootVersion = '3.2.9'
ext.springKafkaVersion = '3.1.6'
ext.openTelemetryVersion = '1.18.0'
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
package com.linkedin.datahub.graphql.types.common.mappers;

import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.generated.DataTransform;
import com.linkedin.datahub.graphql.generated.DataTransformLogic;
import com.linkedin.datahub.graphql.generated.QueryLanguage;
import com.linkedin.datahub.graphql.generated.QueryStatement;
import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;

public class DataTransformLogicMapper
implements ModelMapper<
com.linkedin.common.DataTransformLogic,
com.linkedin.datahub.graphql.generated.DataTransformLogic> {

public static final DataTransformLogicMapper INSTANCE = new DataTransformLogicMapper();

public static DataTransformLogic map(
@Nullable final QueryContext context,
@Nonnull final com.linkedin.common.DataTransformLogic input) {
return INSTANCE.apply(context, input);
}

@Override
public DataTransformLogic apply(
@Nullable final QueryContext context,
@Nonnull final com.linkedin.common.DataTransformLogic input) {

final DataTransformLogic result = new DataTransformLogic();

// Map transforms array using DataTransformMapper
result.setTransforms(
input.getTransforms().stream()
.map(transform -> DataTransformMapper.map(context, transform))
.collect(Collectors.toList()));

return result;
}
}

class DataTransformMapper
implements ModelMapper<
com.linkedin.common.DataTransform, com.linkedin.datahub.graphql.generated.DataTransform> {

public static final DataTransformMapper INSTANCE = new DataTransformMapper();

public static DataTransform map(
@Nullable final QueryContext context,
@Nonnull final com.linkedin.common.DataTransform input) {
return INSTANCE.apply(context, input);
}

@Override
public DataTransform apply(
@Nullable final QueryContext context,
@Nonnull final com.linkedin.common.DataTransform input) {

final DataTransform result = new DataTransform();

// Map query statement if present
if (input.hasQueryStatement()) {
QueryStatement statement =
new QueryStatement(
input.getQueryStatement().getValue(),
QueryLanguage.valueOf(input.getQueryStatement().getLanguage().toString()));
result.setQueryStatement(statement);
}

return result;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package com.linkedin.datahub.graphql.types.common.mappers;

import com.linkedin.data.template.GetMode;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.generated.*;
import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
import com.linkedin.query.QueryProperties;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;

public class QueryPropertiesMapper
implements ModelMapper<
QueryProperties, com.linkedin.datahub.graphql.generated.QueryProperties> {

public static final QueryPropertiesMapper INSTANCE = new QueryPropertiesMapper();

public static com.linkedin.datahub.graphql.generated.QueryProperties map(
@Nullable final QueryContext context, @Nonnull final QueryProperties input) {
return INSTANCE.apply(context, input);
}

@Override
public com.linkedin.datahub.graphql.generated.QueryProperties apply(
@Nullable final QueryContext context, @Nonnull final QueryProperties input) {

final com.linkedin.datahub.graphql.generated.QueryProperties result =
new com.linkedin.datahub.graphql.generated.QueryProperties();

// Map Query Source
result.setSource(QuerySource.valueOf(input.getSource().toString()));

// Map Query Statement
result.setStatement(
new QueryStatement(
input.getStatement().getValue(),
QueryLanguage.valueOf(input.getStatement().getLanguage().toString())));

// Map optional fields
result.setName(input.getName(GetMode.NULL));
result.setDescription(input.getDescription(GetMode.NULL));

// Map origin if present
if (input.hasOrigin() && input.getOrigin() != null) {
result.setOrigin(UrnToEntityMapper.map(context, input.getOrigin()));
}

// Map created audit stamp
AuditStamp created = new AuditStamp();
created.setTime(input.getCreated().getTime());
created.setActor(input.getCreated().getActor(GetMode.NULL).toString());
result.setCreated(created);

// Map last modified audit stamp
AuditStamp lastModified = new AuditStamp();
lastModified.setTime(input.getLastModified().getTime());
lastModified.setActor(input.getLastModified().getActor(GetMode.NULL).toString());
result.setLastModified(lastModified);

return result;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,8 @@ public class DataJobType
BROWSE_PATHS_V2_ASPECT_NAME,
SUB_TYPES_ASPECT_NAME,
STRUCTURED_PROPERTIES_ASPECT_NAME,
FORMS_ASPECT_NAME);
FORMS_ASPECT_NAME,
DATA_TRANSFORM_LOGIC_ASPECT_NAME);
private static final Set<String> FACET_FIELDS = ImmutableSet.of("flow");
private final EntityClient _entityClient;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,7 @@
import static com.linkedin.metadata.Constants.*;

import com.google.common.collect.ImmutableList;
import com.linkedin.common.BrowsePathsV2;
import com.linkedin.common.DataPlatformInstance;
import com.linkedin.common.Deprecation;
import com.linkedin.common.Forms;
import com.linkedin.common.GlobalTags;
import com.linkedin.common.GlossaryTerms;
import com.linkedin.common.InstitutionalMemory;
import com.linkedin.common.Ownership;
import com.linkedin.common.Status;
import com.linkedin.common.SubTypes;
import com.linkedin.common.*;
import com.linkedin.common.urn.Urn;
import com.linkedin.data.DataMap;
import com.linkedin.datahub.graphql.QueryContext;
Expand All @@ -26,15 +17,7 @@
import com.linkedin.datahub.graphql.generated.DataJobProperties;
import com.linkedin.datahub.graphql.generated.Dataset;
import com.linkedin.datahub.graphql.generated.EntityType;
import com.linkedin.datahub.graphql.types.common.mappers.BrowsePathsV2Mapper;
import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper;
import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper;
import com.linkedin.datahub.graphql.types.common.mappers.DeprecationMapper;
import com.linkedin.datahub.graphql.types.common.mappers.FineGrainedLineagesMapper;
import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper;
import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper;
import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper;
import com.linkedin.datahub.graphql.types.common.mappers.SubTypesMapper;
import com.linkedin.datahub.graphql.types.common.mappers.*;
import com.linkedin.datahub.graphql.types.common.mappers.util.SystemMetadataUtils;
import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper;
import com.linkedin.datahub.graphql.types.form.FormsMapper;
Expand Down Expand Up @@ -139,6 +122,9 @@ public DataJob apply(
context, new StructuredProperties(data), entityUrn));
} else if (FORMS_ASPECT_NAME.equals(name)) {
result.setForms(FormsMapper.map(new Forms(data), entityUrn.toString()));
} else if (DATA_TRANSFORM_LOGIC_ASPECT_NAME.equals(name)) {
result.setDataTransformLogic(
DataTransformLogicMapper.map(context, new DataTransformLogic(data)));
}
});

Expand Down
Loading

0 comments on commit 5a7bb17

Please sign in to comment.