From 93e303c65ec0a635edc29e9cbb4c1dbf39ecd5fd Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 11 Jun 2021 18:21:54 -0500 Subject: [PATCH] API: Fix structured dtype cast-safety, promotion, and comparison This PR replaces the old gh-15509 implementing proper type promotion for structured voids. It further fixes the casting safety to consider casts with equivalent field number and matching order as "safe" and if the names, titles, and offsets match as "equiv". The change perculates into the void comparison, and since it fixes the order, it removes the current FutureWarning there as well. This addresses https://github.com/liberfa/pyerfa/issues/77 and replaces gh-15509 (the implementation has changed too much). Fixes gh-15494 (and probably a few more) Co-authored-by: Allan Haldane --- numpy/core/_dtype.py | 16 +++ numpy/core/_internal.py | 42 +++++- numpy/core/src/multiarray/arrayobject.c | 143 +++++++++++-------- numpy/core/src/multiarray/convert_datatype.c | 77 +++++----- numpy/core/src/multiarray/dtypemeta.c | 68 +++++++-- numpy/core/tests/test_deprecations.py | 8 -- numpy/core/tests/test_dtype.py | 6 +- numpy/core/tests/test_multiarray.py | 68 +++++++-- numpy/core/tests/test_nditer.py | 12 +- numpy/core/tests/test_numeric.py | 38 ++--- numpy/lib/tests/test_recfunctions.py | 1 - numpy/testing/tests/test_utils.py | 7 +- 12 files changed, 327 insertions(+), 159 deletions(-) diff --git a/numpy/core/_dtype.py b/numpy/core/_dtype.py index c3a22b1c6bb0..3db80c17eebe 100644 --- a/numpy/core/_dtype.py +++ b/numpy/core/_dtype.py @@ -237,6 +237,11 @@ def _struct_dict_str(dtype, includealignedflag): return ret +def _aligned_offset(offset, alignment): + # round up offset: + return - (-offset // alignment) * alignment + + def _is_packed(dtype): """ Checks whether the structured data type in 'dtype' @@ -249,12 +254,23 @@ def _is_packed(dtype): Duplicates the C `is_dtype_struct_simple_unaligned_layout` function. """ + align = dtype.isalignedstruct + max_alignment = 1 total_offset = 0 for name in dtype.names: fld_dtype, fld_offset, title = _unpack_field(*dtype.fields[name]) + + if align: + total_offset = _aligned_offset(total_offset, fld_dtype.alignment) + max_alignment = max(max_alignment, fld_dtype.alignment) + if fld_offset != total_offset: return False total_offset += fld_dtype.itemsize + + if align: + total_offset = _aligned_offset(total_offset, max_alignment) + if total_offset != dtype.itemsize: return False return True diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py index 8942955f60c6..151791d864bf 100644 --- a/numpy/core/_internal.py +++ b/numpy/core/_internal.py @@ -10,7 +10,7 @@ import platform import warnings -from .multiarray import dtype, array, ndarray +from .multiarray import dtype, array, ndarray, promote_types try: import ctypes except ImportError: @@ -433,6 +433,46 @@ def _copy_fields(ary): 'formats': [dt.fields[name][0] for name in dt.names]} return array(ary, dtype=copy_dtype, copy=True) +def _promote_fields(dt1, dt2): + """ Perform type promotion for two structured dtypes. + + Parameters + ---------- + dt1 : structured dtype + First dtype. + dt2 : structured dtype + Second dtype. + + Returns + ------- + out : dtype + The promoted dtype + + Notes + ----- + If one of the inputs is aligned, the result will be. The titles of + both descriptors must match (point to the same field). + """ + # Both must be structured and have the same names in the same order + if (dt1.names is None or dt2.names is None) or dt1.names != dt2.names: + raise TypeError("invalid type promotion") + + new_fields = [] + for name in dt1.names: + field1 = dt1.fields[name] + field2 = dt2.fields[name] + new_descr = promote_types(field1[0], field2[0]) + # Check that the titles match (if given): + if field1[2:] != field2[2:]: + raise TypeError("invalid type promotion") + if len(field1) == 2: + new_fields.append((name, new_descr)) + else: + new_fields.append(((field1[2], name), new_descr)) + + return dtype(new_fields, align=dt1.isalignedstruct or dt2.isalignedstruct) + + def _getfield_is_safe(oldtype, newtype, offset): """ Checks safety of getfield for object arrays. diff --git a/numpy/core/src/multiarray/arrayobject.c b/numpy/core/src/multiarray/arrayobject.c index 4c20fc1619b5..0cbbc44e7da0 100644 --- a/numpy/core/src/multiarray/arrayobject.c +++ b/numpy/core/src/multiarray/arrayobject.c @@ -1033,31 +1033,83 @@ _void_compare(PyArrayObject *self, PyArrayObject *other, int cmp_op) "Void-arrays can only be compared for equality."); return NULL; } - if (PyArray_HASFIELDS(self)) { - PyObject *res = NULL, *temp, *a, *b; - PyObject *key, *value, *temp2; - PyObject *op; - Py_ssize_t pos = 0; + if (PyArray_TYPE(other) != NPY_VOID) { + PyErr_SetString(PyExc_ValueError, + "Cannot compare structured or void to non-void arrays. " + "(This may return array of False in the future.)"); + return NULL; + } + if (PyArray_HASFIELDS(self) && PyArray_HASFIELDS(other)) { + PyArray_Descr *self_descr = PyArray_DESCR(self); + PyArray_Descr *other_descr = PyArray_DESCR(other); + + /* Use promotion to decide whether the comparison is valid */ + PyArray_Descr *promoted = PyArray_PromoteTypes(self_descr, other_descr); + if (promoted == NULL) { + PyErr_SetString(PyExc_TypeError, + "Cannot compare structured arrays unless they have a " + "common dtype. I.e. `np.result_type(arr1, arr2)` must " + "be defined.\n" + "(This may return array of False in the future.)"); + return NULL; + } + Py_DECREF(promoted); + npy_intp result_ndim = PyArray_NDIM(self) > PyArray_NDIM(other) ? PyArray_NDIM(self) : PyArray_NDIM(other); - op = (cmp_op == Py_EQ ? n_ops.logical_and : n_ops.logical_or); - while (PyDict_Next(PyArray_DESCR(self)->fields, &pos, &key, &value)) { - if (NPY_TITLE_KEY(key, value)) { - continue; - } - a = array_subscript_asarray(self, key); + int field_count = PyTuple_GET_SIZE(self_descr->names); + if (field_count != PyTuple_GET_SIZE(other_descr->names)) { + PyErr_SetString(PyExc_TypeError, + "Cannot compare structured dtypes with different number of " + "fields. (unreachable error please report to NumPy devs)"); + return NULL; + } + + PyObject *op = (cmp_op == Py_EQ ? n_ops.logical_and : n_ops.logical_or); + PyObject *res = NULL; + for (int i = 0; i < field_count; ++i) { + PyObject *fieldname, *temp, *temp2; + + fieldname = PyTuple_GET_ITEM(self_descr->names, i); + PyArrayObject *a = (PyArrayObject *)array_subscript_asarray( + self, fieldname); if (a == NULL) { Py_XDECREF(res); return NULL; } - b = array_subscript_asarray(other, key); + fieldname = PyTuple_GET_ITEM(other_descr->names, i); + PyArrayObject *b = (PyArrayObject *)array_subscript_asarray( + other, fieldname); if (b == NULL) { Py_XDECREF(res); Py_DECREF(a); return NULL; } - temp = array_richcompare((PyArrayObject *)a,b,cmp_op); + /* + * If the fields were subarrays, the dimensions may have changed. + * In that case, the new shape (subarray part) must match exactly. + * (If this is 0, there is no subarray.) + */ + int field_dims_a = PyArray_NDIM(a) - PyArray_NDIM(self); + int field_dims_b = PyArray_NDIM(b) - PyArray_NDIM(other); + if (field_dims_a != field_dims_b || ( + field_dims_a != 0 && /* neither is subarray */ + /* Compare only the added (subarray) dimensions: */ + !PyArray_CompareLists( + PyArray_DIMS(a) + PyArray_NDIM(self), + PyArray_DIMS(b) + PyArray_NDIM(other), + field_dims_a))) { + PyErr_SetString(PyExc_TypeError, + "Cannot compare subarrays with different shapes. " + "(unreachable error, please report to NumPy devs.)"); + Py_DECREF(a); + Py_DECREF(b); + Py_XDECREF(res); + return NULL; + } + + temp = array_richcompare(a, (PyObject *)b, cmp_op); Py_DECREF(a); Py_DECREF(b); if (temp == NULL) { @@ -1142,7 +1194,24 @@ _void_compare(PyArrayObject *self, PyArrayObject *other, int cmp_op) } return res; } + else if (PyArray_HASFIELDS(self) || PyArray_HASFIELDS(other)) { + PyErr_SetString(PyExc_TypeError, + "Cannot compare structured with unstructured void. " + "(This may return array of False in the future.)"); + return NULL; + } else { + /* + * Since arrays absorb subarray descriptors, this path can only be + * reached when both arrays have unstructured voids "V" dtypes. + */ + if (PyArray_ITEMSIZE(self) != PyArray_ITEMSIZE(other)) { + PyErr_SetString(PyExc_TypeError, + "cannot compare unstructured voids of different length. " + "Use bytes to compare. " + "(This may return array of False in the future.)"); + return NULL; + } /* compare as a string. Assumes self and other have same descr->type */ return _strings_richcompare(self, other, cmp_op, 0); } @@ -1345,28 +1414,7 @@ array_richcompare(PyArrayObject *self, PyObject *other, int cmp_op) return Py_NotImplemented; } - _res = PyArray_CheckCastSafety( - NPY_EQUIV_CASTING, - PyArray_DESCR(self), PyArray_DESCR(array_other), NULL); - if (_res < 0) { - PyErr_Clear(); - _res = 0; - } - if (_res == 0) { - /* 2015-05-07, 1.10 */ - Py_DECREF(array_other); - if (DEPRECATE_FUTUREWARNING( - "elementwise == comparison failed and returning scalar " - "instead; this will raise an error or perform " - "elementwise comparison in the future.") < 0) { - return NULL; - } - Py_INCREF(Py_False); - return Py_False; - } - else { - result = _void_compare(self, array_other, cmp_op); - } + result = _void_compare(self, array_other, cmp_op); Py_DECREF(array_other); return result; } @@ -1400,29 +1448,8 @@ array_richcompare(PyArrayObject *self, PyObject *other, int cmp_op) return Py_NotImplemented; } - _res = PyArray_CheckCastSafety( - NPY_EQUIV_CASTING, - PyArray_DESCR(self), PyArray_DESCR(array_other), NULL); - if (_res < 0) { - PyErr_Clear(); - _res = 0; - } - if (_res == 0) { - /* 2015-05-07, 1.10 */ - Py_DECREF(array_other); - if (DEPRECATE_FUTUREWARNING( - "elementwise != comparison failed and returning scalar " - "instead; this will raise an error or perform " - "elementwise comparison in the future.") < 0) { - return NULL; - } - Py_INCREF(Py_True); - return Py_True; - } - else { - result = _void_compare(self, array_other, cmp_op); - Py_DECREF(array_other); - } + result = _void_compare(self, array_other, cmp_op); + Py_DECREF(array_other); return result; } diff --git a/numpy/core/src/multiarray/convert_datatype.c b/numpy/core/src/multiarray/convert_datatype.c index b4a7aad34e6a..929652506b8c 100644 --- a/numpy/core/src/multiarray/convert_datatype.c +++ b/numpy/core/src/multiarray/convert_datatype.c @@ -1015,7 +1015,7 @@ PyArray_FindConcatenationDescriptor( npy_intp n, PyArrayObject **arrays, PyObject *requested_dtype) { if (requested_dtype == NULL) { - return PyArray_LegacyResultType(n, arrays, 0, NULL); + return PyArray_ResultType(n, arrays, 0, NULL); } PyArray_DTypeMeta *common_dtype; @@ -3281,8 +3281,7 @@ can_cast_fields_safety( { Py_ssize_t field_count = PyTuple_Size(from->names); if (field_count != PyTuple_Size(to->names)) { - /* TODO: This should be rejected! */ - return NPY_UNSAFE_CASTING; + return -1; } NPY_CASTING casting = NPY_NO_CASTING; @@ -3294,18 +3293,41 @@ can_cast_fields_safety( if (from_tup == NULL) { return give_bad_field_error(from_key); } - PyArray_Descr *from_base = (PyArray_Descr*)PyTuple_GET_ITEM(from_tup, 0); + PyArray_Descr *from_base = (PyArray_Descr *) PyTuple_GET_ITEM(from_tup, 0); - /* - * TODO: This should use to_key (order), compare gh-15509 by - * by Allan Haldane. And raise an error on failure. - * (Fixing that may also requires fixing/changing promotion.) - */ - PyObject *to_tup = PyDict_GetItem(to->fields, from_key); + /* Check whether the field names match */ + PyObject *to_key = PyTuple_GET_ITEM(to->names, i); + PyObject *to_tup = PyDict_GetItem(to->fields, to_key); if (to_tup == NULL) { - return NPY_UNSAFE_CASTING; + return give_bad_field_error(from_key); + } + PyArray_Descr *to_base = (PyArray_Descr *) PyTuple_GET_ITEM(to_tup, 0); + + int cmp = PyUnicode_Compare(from_key, to_key); + if (error_converting(cmp)) { + return -1; + } + if (cmp != 0) { + /* Field name mismatch, consider this at most SAFE. */ + casting = PyArray_MinCastSafety(casting, NPY_SAFE_CASTING); + } + + /* Also check the title (denote mismatch as SAFE only) */ + PyObject *from_title = from_key; + PyObject *to_title = to_key; + if (PyTuple_GET_SIZE(from_tup) > 2) { + from_title = PyTuple_GET_ITEM(from_tup, 2); + } + if (PyTuple_GET_SIZE(to_tup) > 2) { + to_title = PyTuple_GET_ITEM(to_tup, 2); + } + cmp = PyObject_RichCompareBool(from_title, to_title, Py_EQ); + if (error_converting(cmp)) { + return -1; + } + if (!cmp) { + casting = PyArray_MinCastSafety(casting, NPY_SAFE_CASTING); } - PyArray_Descr *to_base = (PyArray_Descr*)PyTuple_GET_ITEM(to_tup, 0); NPY_CASTING field_casting = PyArray_GetCastInfo( from_base, to_base, NULL, &field_view_off); @@ -3338,39 +3360,26 @@ can_cast_fields_safety( *view_offset = NPY_MIN_INTP; } } - if (*view_offset != 0) { - /* If the calculated `view_offset` is not 0, it can only be "equiv" */ - return PyArray_MinCastSafety(casting, NPY_EQUIV_CASTING); - } /* - * If the itemsize (includes padding at the end), fields, or names - * do not match, this cannot be a view and also not a "no" cast - * (identical dtypes). - * It may be possible that this can be relaxed in some cases. + * If the itemsize (includes padding at the end), does not match, + * this is not a "no" cast (identical dtypes) and may not be viewable. */ if (from->elsize != to->elsize) { /* * The itemsize may mismatch even if all fields and formats match * (due to additional padding). */ - return PyArray_MinCastSafety(casting, NPY_EQUIV_CASTING); - } - - int cmp = PyObject_RichCompareBool(from->fields, to->fields, Py_EQ); - if (cmp != 1) { - if (cmp == -1) { - PyErr_Clear(); + casting = PyArray_MinCastSafety(casting, NPY_EQUIV_CASTING); + if (from->elsize < to->elsize) { + *view_offset = NPY_MIN_INTP; } - return PyArray_MinCastSafety(casting, NPY_EQUIV_CASTING); } - cmp = PyObject_RichCompareBool(from->names, to->names, Py_EQ); - if (cmp != 1) { - if (cmp == -1) { - PyErr_Clear(); - } - return PyArray_MinCastSafety(casting, NPY_EQUIV_CASTING); + else if (*view_offset != 0) { + /* If the calculated `view_offset` is not 0, it can only be "equiv" */ + casting = PyArray_MinCastSafety(casting, NPY_EQUIV_CASTING); } + return casting; } diff --git a/numpy/core/src/multiarray/dtypemeta.c b/numpy/core/src/multiarray/dtypemeta.c index 519b998d7978..2f0dabaca2cd 100644 --- a/numpy/core/src/multiarray/dtypemeta.c +++ b/numpy/core/src/multiarray/dtypemeta.c @@ -399,26 +399,70 @@ void_ensure_canonical(PyArray_Descr *self) static PyArray_Descr * void_common_instance(PyArray_Descr *descr1, PyArray_Descr *descr2) { - /* - * We currently do not support promotion of void types unless they - * are equivalent. - */ - if (!PyArray_CanCastTypeTo(descr1, descr2, NPY_EQUIV_CASTING)) { - if (descr1->subarray == NULL && descr1->names == NULL && - descr2->subarray == NULL && descr2->names == NULL) { + if (descr1->subarray == NULL && descr1->names == NULL && + descr2->subarray == NULL && descr2->names == NULL) { + if (descr1->elsize != descr2->elsize) { PyErr_SetString(PyExc_TypeError, "Invalid type promotion with void datatypes of different " "lengths. Use the `np.bytes_` datatype instead to pad the " "shorter value with trailing zero bytes."); + return NULL; } - else { + Py_INCREF(descr1); + return descr1; + } + + if (descr1->names != NULL && descr2->names != NULL) { + /* If both have fields promoting individual fields may be possible */ + static PyObject *promote_fields_func = NULL; + npy_cache_import("numpy.core._internal", "_promote_fields", + &promote_fields_func); + if (promote_fields_func == NULL) { + return NULL; + } + PyObject *result = PyObject_CallFunctionObjArgs(promote_fields_func, + descr1, descr2, NULL); + if (result == NULL) { + return NULL; + } + if (!PyObject_TypeCheck(result, Py_TYPE(descr1))) { + PyErr_SetString(PyExc_RuntimeError, + "Internal NumPy error: `_promote_fields` did not return " + "a valid descriptor object."); + Py_DECREF(result); + return NULL; + } + return (PyArray_Descr *)result; + } + else if (descr1->subarray != NULL && descr2->subarray != NULL) { + int cmp = PyObject_RichCompareBool( + descr1->subarray->shape, descr2->subarray->shape, Py_EQ); + if (error_converting(cmp)) { + return NULL; + } + if (!cmp) { PyErr_SetString(PyExc_TypeError, - "invalid type promotion with structured datatype(s)."); + "invalid type promotion with subarray datatypes " + "(shape mismatch)."); } - return NULL; + PyArray_Descr *new_base = PyArray_PromoteTypes( + descr1->subarray->base, descr2->subarray->base); + if (new_base == NULL) { + return NULL; + } + + PyArray_Descr *new_descr = PyArray_DescrNew(descr1); + if (new_descr == NULL) { + Py_DECREF(new_base); + return NULL; + } + Py_SETREF(new_descr->subarray->base, new_base); + return new_descr; } - Py_INCREF(descr1); - return descr1; + + PyErr_SetString(PyExc_TypeError, + "invalid type promotion with structured datatype(s)."); + return NULL; } NPY_NO_EXPORT int diff --git a/numpy/core/tests/test_deprecations.py b/numpy/core/tests/test_deprecations.py index c46b294ebcb3..3dd60de91a63 100644 --- a/numpy/core/tests/test_deprecations.py +++ b/numpy/core/tests/test_deprecations.py @@ -185,14 +185,6 @@ def __ne__(self, other): self.assert_deprecated(lambda: np.arange(2) == NotArray()) self.assert_deprecated(lambda: np.arange(2) != NotArray()) - struct1 = np.zeros(2, dtype="i4,i4") - struct2 = np.zeros(2, dtype="i4,i4,i4") - - assert_warns(FutureWarning, lambda: struct1 == 1) - assert_warns(FutureWarning, lambda: struct1 == struct2) - assert_warns(FutureWarning, lambda: struct1 != 1) - assert_warns(FutureWarning, lambda: struct1 != struct2) - def test_array_richcompare_legacy_weirdness(self): # It doesn't really work to use assert_deprecated here, b/c part of # the point of assert_deprecated is to check that when warnings are diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index 1a8e747e1c25..7ce84e28a89a 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -180,11 +180,11 @@ def test_field_order_equality(self): 'formats': ['i4', 'f4'], 'offsets': [0, 4]}) y = np.dtype({'names': ['B', 'A'], - 'formats': ['f4', 'i4'], + 'formats': ['i4', 'f4'], 'offsets': [4, 0]}) assert_equal(x == y, False) - # But it is currently an equivalent cast: - assert np.can_cast(x, y, casting="equiv") + # This is an safe cast (not equiv) due to the different names: + assert np.can_cast(x, y, casting="safe") class TestRecord: diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 6ba90a97f8d0..2f442b1b9e76 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -30,6 +30,7 @@ ) from numpy.testing._private.utils import _no_tracing from numpy.core.tests._locales import CommaDecimalPointLocale +from numpy.lib.recfunctions import repack_fields # Need to test an object that does not fully implement math interface from datetime import timedelta, datetime @@ -1228,21 +1229,15 @@ def test_subarray_comparison(self): # Check that incompatible sub-array shapes don't result to broadcasting x = np.zeros((1,), dtype=[('a', ('f4', (1, 2))), ('b', 'i1')]) y = np.zeros((1,), dtype=[('a', ('f4', (2,))), ('b', 'i1')]) - # This comparison invokes deprecated behaviour, and will probably - # start raising an error eventually. What we really care about in this - # test is just that it doesn't return True. - with suppress_warnings() as sup: - sup.filter(FutureWarning, "elementwise == comparison failed") - assert_equal(x == y, False) + # The main importance is that it does not return True: + with pytest.raises(TypeError): + x == y x = np.zeros((1,), dtype=[('a', ('f4', (2, 1))), ('b', 'i1')]) y = np.zeros((1,), dtype=[('a', ('f4', (2,))), ('b', 'i1')]) - # This comparison invokes deprecated behaviour, and will probably - # start raising an error eventually. What we really care about in this - # test is just that it doesn't return True. - with suppress_warnings() as sup: - sup.filter(FutureWarning, "elementwise == comparison failed") - assert_equal(x == y, False) + # The main importance is that it does not return True: + with pytest.raises(TypeError): + x == y # Check that structured arrays that are different only in # byte-order work @@ -1423,7 +1418,7 @@ def testassign(arr, v): assert_equal(testassign(arr, v1), ans) assert_equal(testassign(arr, v2), ans) assert_equal(testassign(arr, v3), ans) - assert_raises(ValueError, lambda: testassign(arr, v4)) + assert_raises(TypeError, lambda: testassign(arr, v4)) assert_equal(testassign(arr, v5), ans) w[:] = 4 assert_equal(arr, np.array([(1,4),(1,4)], dtype=dt)) @@ -1458,6 +1453,53 @@ def test_multiindex_titles(self): assert_raises(ValueError, lambda : a[['b','b']]) # field exists, but repeated a[['b','c']] # no exception + def test_structured_cast_promotion_fieldorder(self): + # gh-15494 + # dtypes with different field names are not promotable + A = ("a", "i8") + ab = np.array([(1, 2)], dtype=[A, B]) + ba = np.array([(1, 2)], dtype=[B, A]) + assert_raises(TypeError, np.concatenate, ab, ba) + assert_raises(TypeError, np.result_type, ab.dtype, ba.dtype) + assert_raises(TypeError, np.promote_types, ab.dtype, ba.dtype) + + # dtypes with same field names/order but different memory offsets + # and byte-order are promotable to packed nbo. + assert_equal(np.promote_types(ab.dtype, ba[['a', 'b']].dtype), + repack_fields(ab.dtype.newbyteorder('N'))) + + # gh-13667 + # dtypes with different fieldnames but castable field types are castable + assert_equal(np.can_cast(ab.dtype, ba.dtype), True) + assert_equal(ab.astype(ba.dtype).dtype, ba.dtype) + assert_equal(np.can_cast('f8,i8', [('f0', 'f8'), ('f1', 'i8')]), True) + assert_equal(np.can_cast('f8,i8', [('f1', 'f8'), ('f0', 'i8')]), True) + assert_equal(np.can_cast('f8,i8', [('f1', 'i8'), ('f0', 'f8')]), False) + assert_equal(np.can_cast('f8,i8', [('f1', 'i8'), ('f0', 'f8')], + casting='unsafe'), True) + + ab[:] = ba # make sure assignment still works + + # tests of type-promotion of corresponding fields + dt1 = np.dtype([("", "i4")]) + dt2 = np.dtype([("", "i8")]) + assert_equal(np.promote_types(dt1, dt2), np.dtype([('f0', 'i8')])) + assert_equal(np.promote_types(dt2, dt1), np.dtype([('f0', 'i8')])) + assert_raises(TypeError, np.promote_types, dt1, np.dtype([("", "V3")])) + assert_equal(np.promote_types('i4,f8', 'i8,f4'), + np.dtype([('f0', 'i8'), ('f1', 'f8')])) + # test nested case + dt1nest = np.dtype([("", dt1)]) + dt2nest = np.dtype([("", dt2)]) + assert_equal(np.promote_types(dt1nest, dt2nest), + np.dtype([('f0', np.dtype([('f0', 'i8')]))])) + + # note that offsets are lost when promoting: + dt = np.dtype({'names': ['x'], 'formats': ['i4'], 'offsets': [8]}) + a = np.ones(3, dtype=dt) + assert_equal(np.concatenate([a, a]).dtype, np.dtype([('x', 'i4')])) + def test_structured_asarray_is_view(self): # A scalar viewing an array preserves its view even when creating a # new array. This test documents behaviour, it may not be the best diff --git a/numpy/core/tests/test_nditer.py b/numpy/core/tests/test_nditer.py index d96c14e54d7e..0c036861b402 100644 --- a/numpy/core/tests/test_nditer.py +++ b/numpy/core/tests/test_nditer.py @@ -1990,13 +1990,13 @@ def test_iter_buffered_cast_structured_type_failure_with_cleanup(): a = np.array([(1, 2, 3), (4, 5, 6)], dtype=sdt1) for intent in ["readwrite", "readonly", "writeonly"]: - # If the following assert fails, the place where the error is raised - # within nditer may change. That is fine, but it may make sense for - # a new (hard to design) test to replace it. The `simple_arr` is - # designed to require a multi-step cast (due to having fields). - assert np.can_cast(a.dtype, sdt2, casting="unsafe") + # This test was initially designed to test an error at a different + # place, but will now raise earlier to to the cast not being possible: + # `assert np.can_cast(a.dtype, sdt2, casting="unsafe")` fails. + # Without a faulty DType, there is probably probably no reliable + # way to get the initial tested behaviour. simple_arr = np.array([1, 2], dtype="i,i") # requires clean up - with pytest.raises(ValueError): + with pytest.raises(TypeError): nditer((simple_arr, a), ['buffered', 'refs_ok'], [intent, intent], casting='unsafe', op_dtypes=["f,f", sdt2]) diff --git a/numpy/core/tests/test_numeric.py b/numpy/core/tests/test_numeric.py index ad94379115a8..e56e63109941 100644 --- a/numpy/core/tests/test_numeric.py +++ b/numpy/core/tests/test_numeric.py @@ -932,6 +932,25 @@ def test_promote_types_strings(self, swap, string_dtype): # Promote with object: assert_equal(promote_types('O', S+'30'), np.dtype('O')) + @pytest.mark.parametrize(["dtype1", "dtype2"], + [[np.dtype("V6"), np.dtype("V10")], + [np.dtype([("name1", "i8")]), np.dtype([("name2", "i8")])], + ]) + def test_invalid_void_promotion(self, dtype1, dtype2): + # Mainly test structured void promotion, which currently allows + # byte-swapping, but nothing else: + with pytest.raises(TypeError): + np.promote_types(dtype1, dtype2) + + @pytest.mark.parametrize(["dtype1", "dtype2"], + [[np.dtype("V10"), np.dtype("V10")], + [np.dtype([("name1", "i8")])], + [np.dtype("i8,i8"), np.dtype("i8,>i8")], + [np.dtype("i8,i8"), np.dtype("i4,i4")], + ]) + def test_valid_void_promotion(self, dtype1, dtype2): + assert np.promote_types(dtype1, dtype2) == dtype1 + @pytest.mark.parametrize("dtype", list(np.typecodes["All"]) + ["i,i", "S3", "S100", "U3", "U100", rational]) @@ -1015,25 +1034,6 @@ def test_promote_types_metadata(self, dtype1, dtype2): assert res_bs == res assert res_bs.metadata == res.metadata - @pytest.mark.parametrize(["dtype1", "dtype2"], - [[np.dtype("V6"), np.dtype("V10")], - [np.dtype([("name1", "i8")]), np.dtype([("name2", "i8")])], - [np.dtype("i8,i8"), np.dtype("i4,i4")], - ]) - def test_invalid_void_promotion(self, dtype1, dtype2): - # Mainly test structured void promotion, which currently allows - # byte-swapping, but nothing else: - with pytest.raises(TypeError): - np.promote_types(dtype1, dtype2) - - @pytest.mark.parametrize(["dtype1", "dtype2"], - [[np.dtype("V10"), np.dtype("V10")], - [np.dtype([("name1", "i8")])], - [np.dtype("i8,i8"), np.dtype("i8,>i8")], - ]) - def test_valid_void_promotion(self, dtype1, dtype2): - assert np.promote_types(dtype1, dtype2) is dtype1 - def test_can_cast(self): assert_(np.can_cast(np.int32, np.int64)) assert_(np.can_cast(np.float64, complex)) diff --git a/numpy/lib/tests/test_recfunctions.py b/numpy/lib/tests/test_recfunctions.py index 2f3c14df31f0..9b2506a7c0fd 100644 --- a/numpy/lib/tests/test_recfunctions.py +++ b/numpy/lib/tests/test_recfunctions.py @@ -835,7 +835,6 @@ def test_duplicate_keys(self): b = np.ones(3, dtype=[('c', 'u1'), ('b', 'f4'), ('a', 'i4')]) assert_raises(ValueError, join_by, ['a', 'b', 'b'], a, b) - @pytest.mark.xfail(reason="See comment at gh-9343") def test_same_name_different_dtypes_key(self): a_dtype = np.dtype([('key', 'S5'), ('value', '