Skip to content

Commit

Permalink
YDA-5694: replace msi_add_avu microservice with atomic metadata opera…
Browse files Browse the repository at this point in the history 
…tions
  • Loading branch information
@kaur16
kaur16 authored Aug 29, 2024
1 parent 5936bd7 commit 4174135
Show file tree
Hide file tree
Showing 3 changed files with 159 additions and 41 deletions.
144 changes: 107 additions & 37 deletions meta.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -377,68 +377,138 @@ def update_index_metadata(ctx, path, metadata, creation_time, data_package):
"""Update the index attributes for JSON metadata."""
msi.coll_create(ctx, path, "", irods_types.BytesBuf())
ctx.msi_rmw_avu('-C', path, '%', '%', constants.UUFLATINDEX)
avu_op = "add"
avu_unit = constants.UUFLATINDEX
metadata_operations = {
"entity_name": path,
"entity_type": "collection",
"operations": []
}

for creator in metadata['Creator']:
name = creator['Name']
if 'Given_Name' in name and 'Family_Name' in name:
ctx.msi_add_avu('-C', path, 'Creator',
name['Given_Name'] + ' ' + name['Family_Name'],
constants.UUFLATINDEX)
metadata_operations['operations'].append({
"operation": avu_op,
"attribute": "Creator",
"value": name['Given_Name'] + ' ' + name['Family_Name'],
"units": avu_unit
})
if 'Owner_Role' in creator:
ctx.msi_add_avu('-C', path, 'Owner_Role', creator['Owner_Role'],
constants.UUFLATINDEX)
metadata_operations['operations'].append({
"operation": avu_op,
"attribute": "Owner_Role",
"value": creator['Owner_Role'],
"units": avu_unit
})

if 'Contributor' in metadata:
for contributor in metadata['Contributor']:
if 'Name' in contributor:
name = contributor['Name']
if 'Given_Name' in name and 'Family_Name' in name:
ctx.msi_add_avu('-C', path, 'Contributor',
name['Given_Name'] + ' ' + name['Family_Name'],
constants.UUFLATINDEX)
metadata_operations['operations'].append({
"operation": avu_op,
"attribute": "Contributor",
"value": name['Given_Name'] + ' ' + name['Family_Name'],
"units": avu_unit
})

if 'Tag' in metadata:
for tag in metadata['Tag']:
ctx.msi_add_avu('-C', path, 'Tag', tag,
constants.UUFLATINDEX)

ctx.msi_add_avu('-C', path, 'Title', metadata['Title'],
constants.UUFLATINDEX)
ctx.msi_add_avu('-C', path, 'Description', metadata['Description'],
constants.UUFLATINDEX)
ctx.msi_add_avu('-C', path, 'Data_Access_Restriction',
metadata['Data_Access_Restriction'], constants.UUFLATINDEX)
metadata_operations['operations'].append({
"operation": avu_op,
"attribute": "Tag",
"value": tag,
"units": avu_unit
})

extend_operations = [
{
"operation": avu_op,
"attribute": "Title",
"value": metadata['Title'],
"units": avu_unit
},
{
"operation": avu_op,
"attribute": "Description",
"value": metadata['Description'],
"units": avu_unit
},
{
"operation": avu_op,
"attribute": "Data_Access_Restriction",
"value": metadata['Data_Access_Restriction'],
"units": avu_unit
},
{
"operation": avu_op,
"attribute": "Creation_Time",
"value": creation_time,
"units": avu_unit
},
{
"operation": avu_op,
"attribute": "Creation_Year",
"value": str(datetime.fromtimestamp(int(creation_time)).year),
"units": avu_unit
}]

metadata_operations['operations'].extend(extend_operations)

if 'Research_Group' in metadata:
ctx.msi_add_avu('-C', path, 'Research_Group',
metadata['Research_Group'], constants.UUFLATINDEX)
metadata_operations['operations'].append({
"operation": avu_op,
"attribute": "Research_Group",
"value": metadata['Research_Group'],
"units": avu_unit
})
if 'Collection_Name' in metadata:
ctx.msi_add_avu('-C', path, 'Collection_Name',
metadata['Collection_Name'], constants.UUFLATINDEX)
metadata_operations['operations'].append({
"operation": avu_op,
"attribute": "Collection_Name",
"value": metadata['Collection_Name'],
"units": avu_unit
})
if 'Collected' in metadata:
if 'Start_Date' in metadata['Collected']:
ctx.msi_add_avu('-C', path, 'Collected_Start_Year',
metadata['Collected']['Start_Date'][:4],
constants.UUFLATINDEX)
metadata_operations['operations'].append({
"operation": avu_op,
"attribute": "Collected_Start_Year",
"value": metadata['Collected']['Start_Date'][:4],
"units": avu_unit
})
if 'End_Date' in metadata['Collected']:
ctx.msi_add_avu('-C', path, 'Collected_End_Year',
metadata['Collected']['End_Date'][:4],
constants.UUFLATINDEX)
metadata_operations['operations'].append({
"operation": avu_op,
"attribute": "Collected_End_Year",
"value": metadata['Collected']['End_Date'][:4],
"units": avu_unit
})

if 'GeoLocation' in metadata:
for geoLocation in metadata['GeoLocation']:
if 'Description_Spatial' in geoLocation:
ctx.msi_add_avu('-C', path, 'Description_Spatial', geoLocation['Description_Spatial'],
constants.UUFLATINDEX)

ctx.msi_add_avu('-C', path, 'Creation_Time', creation_time,
constants.UUFLATINDEX)
ctx.msi_add_avu('-C', path, 'Creation_Year',
str(datetime.fromtimestamp(int(creation_time)).year),
constants.UUFLATINDEX)
metadata_operations['operations'].append({
"operation": avu_op,
"attribute": "Description_Spatial",
"value": geoLocation['Description_Spatial'],
"units": avu_unit
})

if config.enable_data_package_reference:
ctx.msi_add_avu('-C', path, 'Data_Package_Reference', data_package,
constants.UUFLATINDEX)
metadata_operations['operations'].append({
"operation": "add",
"attribute": "Data_Package_Reference",
"value": data_package,
"units": avu_unit
})

if avu.apply_atomic_operations(ctx, metadata_operations):
log.write(ctx, 'update_index_metadata: Metadata index update successful on path {}'.format(path))
else:
log.write(ctx, 'update_index_metadata: Metadata index update unsuccessful on path {}'.format(path))


def ingest_metadata_vault(ctx, path):
Expand Down
42 changes: 39 additions & 3 deletions replication.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,19 @@ def replicate_asynchronously(ctx, path, source_resource, target_resource):

if not already_has_avu:
# Can't use mod_avu/set here (instead of add_avu) because it would be blocked by metadata policies.
msi.add_avu(ctx, '-d', path, replication_avu_name, replication_avu_value, "")
add_operation = {
"entity_name": path,
"entity_type": "data_object",
"operations": [
{
"operation": "add",
"attribute": replication_avu_name,
"value": replication_avu_value,
"units": ""
}
]
}
avu.apply_atomic_operations(ctx, add_operation)
except msi.Error as e:
if "-817000" in str(e):
# CAT_UNKNOWN_FILE: object has been removed in the mean time. No need to replicate it anymore.
Expand Down Expand Up @@ -132,7 +144,19 @@ def rule_replicate_batch(ctx, verbose, balance_id_min, balance_id_max, batch_siz
# Not replicable.
log.write(ctx, "ERROR - Invalid replication data for {}".format(path))
try:
ctx.msi_add_avu('-d', path, errorattr, "Invalid,Invalid", "")
add_operation = {
"entity_name": path,
"entity_type": "data_object",
"operations": [
{
"operation": "add",
"attribute": errorattr,
"value": "Invalid,Invalid",
"units": ""
}
]
}
avu.apply_atomic_operations(ctx, add_operation)
except Exception:
pass

Expand Down Expand Up @@ -184,7 +208,19 @@ def rule_replicate_batch(ctx, verbose, balance_id_min, balance_id_max, batch_siz
except msi.Error as e:
log.write(ctx, 'ERROR - The file could not be replicated: {}'.format(str(e)))
try:
ctx.msi_add_avu('-d', path, errorattr, "{},{}".format(from_path, to_path), "")
add_operation = {
"entity_name": path,
"entity_type": "data_object",
"operations": [
{
"operation": "add",
"attribute": errorattr,
"value": "{},{}".format(from_path, to_path),
"units": ""
}
]
}
avu.apply_atomic_operations(ctx, add_operation)
except Exception:
pass

Expand Down
14 changes: 13 additions & 1 deletion revisions.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -288,7 +288,19 @@ def resource_modified_post_revision(ctx, resource, zone, path):
offset=0, limit=1, output=genquery.AS_LIST))) > 0

if not already_has_avu:
msi.add_avu(ctx, '-d', path, revision_avu_name, revision_avu_value, "")
add_operation = {
"entity_name": path,
"entity_type": "data_object",
"operations": [
{
"operation": "add",
"attribute": revision_avu_name,
"value": revision_avu_value,
"units": ""
}
]
}
avu.apply_atomic_operations(ctx, add_operation)

except msi.Error as e:
if "-817000" in str(e):
Expand Down

0 comments on commit 4174135

Please sign in to comment.