diff --git a/build.py b/build.py
index c065a6a..14c8511 100644
--- a/build.py
+++ b/build.py
@@ -26,14 +26,9 @@ def build(setup_kwargs: Any) -> Any:
     setup_oj()
 
     # Determine if system is big endian or not
-    macros = [
-        ("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION"),
-        # ("USE_JPIP", "0"),
-    ]
+    macros = [("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION")]
     if unpack("h", b"\x00\x01")[0] == 1:
-        # macros.append(("ON_BE_SYSTEM", None))
-        macros.append(("OPJ_BIG_ENDIAN", None))
-        # macros.append(("__BIG_ENDIAN__", None))
+        macros.append(("PYOJ_BIG_ENDIAN", None))
 
     ext = Extension(
         "_openjpeg",
@@ -151,14 +146,6 @@ def setup_oj() -> None:
             f.write("\n")
             f.write("#define USE_JPIP 0")
 
-    # Use our own OPJ_BIG_ENDIAN macro
-    if os.path.exists(INTERFACE_SRC / "opj_config_private.h"):
-        with open(INTERFACE_SRC / "opj_config_private.h", "r") as f:
-            lines = f.readlines()
-
-        with open(INTERFACE_SRC / "opj_config_private.h", "w") as f:
-            f.writelines(lines[:-5])
-
 
 def reset_oj() -> None:
     # Restore submodule to original state
diff --git a/lib/interface/decode.c b/lib/interface/decode.c
index d0ee406..3087ec9 100644
--- a/lib/interface/decode.c
+++ b/lib/interface/decode.c
@@ -635,24 +635,25 @@ extern int Decode(PyObject* fd, unsigned char *out, int codec_format)
                 {
                     u16.val = (unsigned short)(*p_component[ii]);
                     // Ensure little endian output
-#ifdef OPJ_BIG_ENDIAN
-                    *out = u16.vals[1];
-                    out++;
-                    *out = u16.vals[0];
-                    out++;
-#else
-                    *out = u16.vals[0];
-                    out++;
-                    *out = u16.vals[1];
-                    out++;
-#endif
+                    #ifdef PYOJ_BIG_ENDIAN
+                        *out = u16.vals[1];
+                        out++;
+                        *out = u16.vals[0];
+                        out++;
+                    #else
+                        *out = u16.vals[0];
+                        out++;
+                        *out = u16.vals[1];
+                        out++;
+                    #endif
+
                     p_component[ii]++;
                 }
             }
         }
     } else if (precision <= 32) {
         union {
-            unsigned long val;
+            OPJ_INT32 val;
             unsigned char vals[4];
         } u32;
 
@@ -663,27 +664,27 @@ extern int Decode(PyObject* fd, unsigned char *out, int codec_format)
             {
                 for (ii = 0; ii < NR_COMPONENTS; ii++)
                 {
-                    u32.val = (unsigned long)(*p_component[ii]);
+                    u32.val = (OPJ_INT32)(*p_component[ii]);
                     // Ensure little endian output
-#ifdef OPJ_BIG_ENDIAN
-                    *out = u32.vals[3];
-                    out++;
-                    *out = u32.vals[2];
-                    out++;
-                    *out = u32.vals[1];
-                    out++;
-                    *out = u32.vals[0];
-                    out++;
-#else
-                    *out = u32.vals[0];
-                    out++;
-                    *out = u32.vals[1];
-                    out++;
-                    *out = u32.vals[2];
-                    out++;
-                    *out = u32.vals[3];
-                    out++;
-#endif
+                    #ifdef PYOJ_BIG_ENDIAN
+                        *out = u32.vals[3];
+                        out++;
+                        *out = u32.vals[2];
+                        out++;
+                        *out = u32.vals[1];
+                        out++;
+                        *out = u32.vals[0];
+                        out++;
+                    #else
+                        *out = u32.vals[0];
+                        out++;
+                        *out = u32.vals[1];
+                        out++;
+                        *out = u32.vals[2];
+                        out++;
+                        *out = u32.vals[3];
+                        out++;
+                    #endif
 
                     p_component[ii]++;
                 }
diff --git a/openjpeg/tests/test_encode.py b/openjpeg/tests/test_encode.py
index afb846f..d5460c2 100644
--- a/openjpeg/tests/test_encode.py
+++ b/openjpeg/tests/test_encode.py
@@ -425,21 +425,16 @@ def test_lossless_unsigned(self):
             assert out.dtype.kind == "u"
             assert np.array_equal(arr, out)
 
-    # @pytest.mark.skipif(sys.byteorder == "big", reason="Running on BE system")
     def test_lossless_unsigned_u4(self):
         """Test encoding unsigned data for bit-depth 17-32"""
-        dtype = "<u4" if sys.byteorder == "little" else ">u4"
         rows = 123
         cols = 234
         planes = 3
         for bit_depth in range(17, 25):
             maximum = 2**bit_depth - 1
-            arr = np.random.randint(0, high=maximum + 1, size=(rows, cols), dtype=dtype)
-            # if sys.byteorder == "big":
-            #     arr = arr.byteswap().astype("<u4")
+            arr = np.random.randint(0, high=maximum + 1, size=(rows, cols), dtype="u4")
             buffer = encode_array(arr, photometric_interpretation=PI.MONOCHROME2)
             out = decode(buffer)
-            print(bit_depth, out.min(), out.max())
 
             param = parse_j2k(buffer)
             assert param["precision"] == bit_depth
@@ -451,7 +446,7 @@ def test_lossless_unsigned_u4(self):
             assert np.array_equal(arr, out)
 
             arr = np.random.randint(
-                0, high=maximum + 1, size=(rows, cols, planes), dtype=dtype
+                0, high=maximum + 1, size=(rows, cols, planes), dtype="u4"
             )
             buffer = encode_array(arr, photometric_interpretation=PI.RGB, use_mct=False)
             out = decode(buffer)
@@ -521,8 +516,7 @@ def test_lossless_signed(self):
             assert out.dtype.kind == "i"
             assert np.array_equal(arr, out)
 
-    # @pytest.mark.skipif(sys.byteorder == "big", reason="Running on BE system")
-    def test_lossless_signed_u4(self):
+    def test_lossless_signed_i4(self):
         """Test encoding signed data for bit-depth 17-32"""
         rows = 123
         cols = 234
@@ -531,7 +525,7 @@ def test_lossless_signed_u4(self):
             maximum = 2 ** (bit_depth - 1) - 1
             minimum = -(2 ** (bit_depth - 1))
             arr = np.random.randint(
-                low=minimum, high=maximum + 1, size=(rows, cols), dtype="<i4"
+                low=minimum, high=maximum + 1, size=(rows, cols), dtype="i4"
             )
             buffer = encode_array(arr, photometric_interpretation=PI.MONOCHROME2)
             out = decode(buffer)
@@ -546,7 +540,7 @@ def test_lossless_signed_u4(self):
             assert np.array_equal(arr, out)
 
             arr = np.random.randint(
-                low=minimum, high=maximum + 1, size=(rows, cols, planes), dtype="<i4"
+                low=minimum, high=maximum + 1, size=(rows, cols, planes), dtype="i4"
             )
             buffer = encode_array(arr, photometric_interpretation=PI.RGB, use_mct=False)
             out = decode(buffer)
@@ -1227,7 +1221,7 @@ def test_lossless_unsigned_u4(self):
         rows = 123
         cols = 234
         planes = 3
-        for bit_depth in range(24, 32):
+        for bit_depth in range(17, 25):
             maximum = 2**bit_depth - 1
             arr = np.random.randint(0, high=maximum + 1, size=(rows, cols), dtype="u4")
             if sys.byteorder == "big":
@@ -1243,14 +1237,6 @@ def test_lossless_unsigned_u4(self):
                 photometric_interpretation=PI.MONOCHROME2,
             )
             out = decode(buffer)
-            print(bit_depth, arr.min(), arr.max())
-            print(bit_depth, out.min(), out.max())
-            import matplotlib.pyplot as plt
-
-            fig, (ax1, ax2) = plt.subplots(1, 2)
-            ax1.imshow(arr)
-            ax2.imshow(out)
-            plt.show()
 
             param = parse_j2k(buffer)
             assert param["precision"] == bit_depth
@@ -1462,12 +1448,15 @@ def test_lossless_signed_i4(self):
         rows = 123
         cols = 234
         planes = 3
-        for bit_depth in range(24, 32):
+        for bit_depth in range(17, 25):
             maximum = 2 ** (bit_depth - 1) - 1
             minimum = -(2 ** (bit_depth - 1))
             arr = np.random.randint(
-                low=minimum, high=maximum + 1, size=(rows, cols), dtype="<i4"
+                low=minimum, high=maximum + 1, size=(rows, cols), dtype="i4"
             )
+            if sys.byteorder == "big":
+                arr = arr.byteswap().view("<i4")
+
             buffer = encode_buffer(
                 arr.tobytes(),
                 cols,
@@ -1478,14 +1467,7 @@ def test_lossless_signed_i4(self):
                 photometric_interpretation=PI.MONOCHROME2,
             )
             out = decode(buffer)
-            print(bit_depth, arr.min(), arr.max())
-            print(bit_depth, out.min(), out.max())
-            import matplotlib.pyplot as plt
 
-            fig, (ax1, ax2) = plt.subplots(1, 2)
-            ax1.imshow(arr)
-            ax2.imshow(out)
-            plt.show()
 
             param = parse_j2k(buffer)
             assert param["precision"] == bit_depth
@@ -1497,8 +1479,11 @@ def test_lossless_signed_i4(self):
             assert np.array_equal(arr, out)
 
             arr = np.random.randint(
-                low=minimum, high=maximum + 1, size=(rows, cols, planes), dtype="<i4"
+                low=minimum, high=maximum + 1, size=(rows, cols, planes), dtype="i4"
             )
+            if sys.byteorder == "big":
+                arr = arr.byteswap().view("<i4")
+
             buffer = encode_buffer(
                 arr.tobytes(),
                 cols,