New property or item status change script

CHANGELOG.rst

@@ -7,6 +7,14 @@ Change Log
 ------
 
 
+3.1.0
+=====
+
+`PR:113 add new delete/patch script <https://github.com/4dn-dcic/dcicwrangling/pull/113>_`
+
+* added a new script to facilitating deleting the same field(s) for multiple items or setting the status of the items to 'deleted'
+* added tests 
+
 3.0.2
 =====
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicwrangling"
-version = "3.0.2"
+version = "3.1.0"
 description = "Scripts and Jupyter notebooks for 4DN wrangling"
 authors = ["4DN-DCIC Team <[email protected]>"]
 license = "MIT"

scripts/delete_item_or_fields.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+'''
+Given a list of item IDs or search result will:
+    1. given one of more values in the '--fields' options delete those fields from the item
+    2. if no fields option then set the status of the item(s) to deleted
+Useful for deleting multiple fields or changing item status to deleted
+NOTE: can use patch_field_for_many_items script to delete a single field for many items
+but this script is more direct/flexible in some ways
+'''
+import argparse
+from dcicutils.ff_utils import get_metadata, delete_metadata, delete_field
+from functions import script_utils as scu
+
+
+def get_args(args):
+    parser = argparse.ArgumentParser(
+        parents=[scu.create_input_arg_parser(), scu.create_ff_arg_parser()],
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument('--fields',
+                        nargs='+',
+                        help="With this option these fields will be removed from the items.")
+    args = parser.parse_args(args)
+    return args
+
+
+def main():  # pragma: no cover
+    args = get_args()
+    auth = scu.authenticate(key=args.key, keyfile=args.keyfile, env=args.env)
+    dry_run = True
+    if args.dbupdate:
+        dry_run = False
+
+    print('#', auth.get('server'))
+    id_list = scu.get_item_ids_from_args(args.input, auth, args.search)
+    del_flds = None
+    if args.fields:
+        fields = args.fields
+        del_flds = ','.join(fields)
+    problems = []
+    for iid in id_list:
+        try:
+            get_metadata(iid, auth, add_on='frame=object')
+        except Exception:
+            problems.append(iid)
+            continue
+
+        if del_flds:
+            # we have field(s) that we want to delete from provided items
+            print(f"Will delete {del_flds} from {iid}")
+            if dry_run:
+                print("DRY RUN")
+            else:
+                try:
+                    res = delete_field(iid, del_flds, auth)
+                    status = res.get('status')
+                    if status.lower() == 'success':
+                        print(status)
+                    else:
+                        print(res)
+                        problems.append(iid)
+                except Exception as e:
+                    print(f"PROBLEM: {e}")
+                    problems.append(iid)
+        else:
+            # we want to set the status of the items in id_list to deleted
+            print(f"Will set status of {iid} to DELETED")
+            if dry_run:
+                print("DRY RUN")
+            else:
+                try:
+                    res = delete_metadata(iid, auth)
+                    status = res.get('status')
+                    if status.lower() == 'success':
+                        print(status)
+                    else:
+                        print(res)
+                        problems.append(iid)
+                except Exception as e:
+                    print(f"PROBLEM: {e}")
+                    problems.append(iid)
+
+    if problems:
+        print('THERE WAS A PROBLEM DELETING METADATA FOR THE FOLLOWING:')
+        for p in problems:
+            print(p)
+
+
+if __name__ == '__main__':
+    main()

scripts/patch_field_for_many_items.py

@@ -1,6 +1,6 @@
 import sys
 import argparse
-from dcicutils.ff_utils import get_authentication_with_server, patch_metadata, delete_field
+from dcicutils.ff_utils import patch_metadata, delete_field
 from functions import script_utils as scu
 
 
@@ -13,7 +13,8 @@ def get_args(args):
                         help="The field to update.")
     parser.add_argument('value',
                         help="The value(s) to update. Array fields need \"''\" surround \
-                        even if only a single value i.e. \"'value here'\" or \"'v1' 'v2'\"")
+                        even if only a single value i.e. \"'value here'\" or \"'v1' 'v2'\" \
+                        NOTE: can delete field by passing val *delete*")
     parser.add_argument('--isarray',
                         default=False,
                         action='store_true',

tests/test_delete_item_or_fields.py

@@ -0,0 +1,149 @@
+import pytest
+from scripts import delete_item_or_fields as diof
+
+
+def test_diof_get_args_required_default():
+    defaults = {
+        'dbupdate': False,
+        'env': None,
+        'key': None,
+        'keyfile': 'keypairs.json',
+        'search': False,
+    }
+    args = diof.get_args(['i'])
+    for k, v in defaults.items():
+        assert getattr(args, k) == v
+    assert args.input == ['i']
+
+
+def test_diof_get_args_missing_required(capsys):
+    with pytest.raises(SystemExit) as pe:
+        diof.get_args([])
+        out = capsys.readouterr()[0]
+        assert 'error: the following arguments are required: input' in out
+    assert pe.type == SystemExit
+    assert pe.value.code == 2
+
+
+class MockedNamespace(object):
+    def __init__(self, dic):
+        for k, v in dic.items():
+            setattr(self, k, v)
+
+
+@pytest.fixture
+def mocked_args_dbupd_is_false():
+    return MockedNamespace(
+        {
+            'key': None,
+            'keyfile': None,
+            'env': 'prod',
+            'dbupdate': False,
+            'search': False,
+            'input': ['id1', 'id2'],
+            'fields': None
+        }
+    )
+
+
+@pytest.fixture
+def mocked_args_dbupd_is_true():
+    return MockedNamespace(
+        {
+            'key': None,
+            'keyfile': None,
+            'env': 'prod',
+            'dbupdate': True,
+            'search': False,
+            'input': ['id1', 'id2'],
+            'fields': None
+        }
+    )
+
+
+@pytest.fixture
+def mocked_args_w_fields_dry_run():
+    return MockedNamespace(
+        {
+            'key': None,
+            'keyfile': None,
+            'env': 'prod',
+            'dbupdate': False,
+            'search': False,
+            'input': ['id1', 'id2'],
+            'fields': ['aliases'],
+        }
+    )
+
+
+@pytest.fixture
+def mocked_args_w_fields_dbudate():
+    return MockedNamespace(
+        {
+            'key': None,
+            'keyfile': None,
+            'env': 'prod',
+            'dbupdate': True,
+            'search': False,
+            'input': ['id1', 'id2'],
+            'fields': ['aliases'],
+        }
+    )
+def test_diof_main_dryrun_no_fields(mocker, capsys, mocked_args_dbupd_is_false, auth):
+    iids = ['id1', 'id2']
+    mocker.patch('scripts.delete_item_or_fields.get_args', return_value=mocked_args_dbupd_is_false)
+    mocker.patch('scripts.delete_item_or_fields.scu.authenticate', return_value=auth)
+    mocker.patch('scripts.delete_item_or_fields.scu.get_item_ids_from_args', return_value=iids)
+    mocker.patch('scripts.delete_item_or_fields.get_metadata', side_effect=[None, None])
+    diof.main()
+    out = capsys.readouterr()[0]
+    for i in iids:
+        s = "Will set status of %s to DELETED\nDRY RUN" % i
+        assert s in out
+
+
+def test_diof_main_dryrun_w_fields(mocker, capsys, mocked_args_w_fields_dry_run, auth):
+    iids = ['id1', 'id2']
+    mocker.patch('scripts.delete_item_or_fields.get_args', return_value=mocked_args_w_fields_dry_run)
+    mocker.patch('scripts.delete_item_or_fields.scu.authenticate', return_value=auth)
+    mocker.patch('scripts.delete_item_or_fields.scu.get_item_ids_from_args', return_value=iids)
+    mocker.patch('scripts.delete_item_or_fields.get_metadata', side_effect=[None, None])
+    diof.main()
+    out = capsys.readouterr()[0]
+    for i in iids:
+        s = "Will delete aliases from %s\nDRY RUN" % i
+        assert s in out
+
+
+def test_diof_main_dbupdate_delstatus_items(mocker, capsys, mocked_args_dbupd_is_true, auth):
+    iids = ['id1', 'id2']
+    resp1 = {'status': 'success'}
+    resp2 = {'status': 'error', 'description': "access denied"}
+    mocker.patch('scripts.delete_item_or_fields.get_args', return_value=mocked_args_dbupd_is_true)
+    mocker.patch('scripts.delete_item_or_fields.scu.authenticate', return_value=auth)
+    mocker.patch('scripts.delete_item_or_fields.scu.get_item_ids_from_args', return_value=iids)
+    mocker.patch('scripts.delete_item_or_fields.get_metadata', side_effect=[None, None])
+    mocker.patch('scripts.delete_item_or_fields.delete_metadata', side_effect=[resp1, resp2])
+    diof.main()
+    out = capsys.readouterr()[0]
+    s1 = "Will set status of %s to DELETED\nsuccess" % iids[0]
+    s2 = "Will set status of %s to DELETED\n{'status': 'error', 'description': 'access denied'}" %iids[1]
+    assert s1 in out
+    assert s2 in out
+
+
+def test_diof_main_dbupdate_delfields(mocker, capsys, mocked_args_w_fields_dbudate, auth):
+    iids = ['id1', 'id2']
+    resp1 = {'status': 'success'}
+    resp2 = {'status': 'error', 'description': "property not found"}
+    mocker.patch('scripts.delete_item_or_fields.get_args', return_value=mocked_args_w_fields_dbudate)
+    mocker.patch('scripts.delete_item_or_fields.scu.authenticate', return_value=auth)
+    mocker.patch('scripts.delete_item_or_fields.scu.get_item_ids_from_args', return_value=iids)
+    mocker.patch('scripts.delete_item_or_fields.get_metadata', side_effect=[None, None])
+    mocker.patch('scripts.delete_item_or_fields.delete_field', side_effect=[resp1, resp2])
+    diof.main()
+    out = capsys.readouterr()[0]
+    s1 = "Will delete aliases from %s\nsuccess" % iids[0]
+    s2 = "Will delete aliases from %s\n{'status': 'error', 'description': 'property not found'}" %iids[1]
+    assert s1 in out
+    assert s2 in out