From 2e6542fd2ee1b8c16f53717cf8cd0f60d282ec1b Mon Sep 17 00:00:00 2001 From: DAMIE Marc Date: Sat, 22 Apr 2023 17:30:20 +0200 Subject: [PATCH 1/6] Implement np_shuffle --- .gitignore | 3 ++ mpyc/random.py | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/.gitignore b/.gitignore index 7bbc71c0..b660602d 100644 --- a/.gitignore +++ b/.gitignore @@ -99,3 +99,6 @@ ENV/ # mypy .mypy_cache/ + +# vscode +.vscode diff --git a/mpyc/random.py b/mpyc/random.py index 1cfb5d81..d7bd4708 100644 --- a/mpyc/random.py +++ b/mpyc/random.py @@ -322,3 +322,79 @@ def uniform(sectype, a, b): s = math.copysign(1, b - a) return a + _randbelow(sectype, round(abs(a - b) * 2**f)) * s * 2**-f + + +@asyncoro.mpc_coro +async def np_random_unit_vector(sectype, n): + """Uniformly random secret rotation of [1] + [0]*(n-1). + + Expected number of secret random bits needed is ceil(log_2 n) + c, + with c a small constant, c < 3. + """ + + if issubclass(sectype, runtime.SecureObject): + await runtime.returnType((sectype.array, True, (n,))) + else: + await runtime.returnType(asyncoro.Future) + + if n == 1: + return runtime.np_fromlist([sectype(1)]) + + b = n - 1 + k = b.bit_length() + x = runtime.np_random_bits(sectype, k) + + i = k - 1 + u = runtime.np_fromlist([x[i], 1 - x[i]]) + while i: + i -= 1 + if (b >> i) & 1: + v = x[i] * u + v = runtime.np_hstack((v, u - v)) + u = v + elif await runtime.output(u[0] * x[i]): # TODO: mul_public + # restart, keeping unused secret random bits x[:i] + x = runtime.np_hstack((x[:i], np_random_unit_vector(sectype, k - i))) + i = k - 1 + u = runtime.np_fromlist([x[i], 1 - x[i]]) + else: + v = x[i] * u[1:] + v = runtime.np_hstack((v, u[1:] - v)) + u = runtime.np_hstack((u[:1], v)) + return u + + +def np_shuffle(a, axis=None): + """Shuffle numpy-like array x secretly in-place, and return None. + + Given array x may contain public or secret elements. + """ + sectype = type(a).sectype + if axis is None: + axis = 0 + + if axis >= len(a.shape): + raise ValueError("Invalid axis") + + x = runtime.np_copy(a) + + if axis != 0: + x = runtime.np_swapaxes(x, 0, axis) + + n = x.shape[0] + + for i in range(n - 1): + u = runtime.np_transpose(np_random_unit_vector(sectype, n - i)) + x_u = runtime.np_matmul(u, x[i:]) + if len(x.shape) > 1: + d = runtime.np_outer(u, (x[i] - x_u)) + x = runtime.np_vstack((x[:i, ...], runtime.np_add(x[i:, ...], d))) + else: + d = u * (x[i] - x_u) + x = runtime.np_hstack((x[:i, ...], runtime.np_add(x[i:, ...], d))) + runtime.np_update(x, i, x_u) + + if axis != 0: + x = runtime.np_swapaxes(x, 0, axis) + + runtime.np_update(a, range(a.shape[0]), x) From 05fcf499de2c43a499f8b84f3cb8a4ecc0b662cf Mon Sep 17 00:00:00 2001 From: DAMIE Marc Date: Sat, 22 Apr 2023 22:31:29 +0200 Subject: [PATCH 2/6] Unit test for np_shuffle --- mpyc/random.py | 2 +- tests/test_random.py | 26 +++++++++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/mpyc/random.py b/mpyc/random.py index d7bd4708..4f962943 100644 --- a/mpyc/random.py +++ b/mpyc/random.py @@ -373,7 +373,7 @@ def np_shuffle(a, axis=None): if axis is None: axis = 0 - if axis >= len(a.shape): + if axis >= len(a.shape) or axis <0: raise ValueError("Invalid axis") x = runtime.np_copy(a) diff --git a/tests/test_random.py b/tests/test_random.py index 489ca276..3941342b 100644 --- a/tests/test_random.py +++ b/tests/test_random.py @@ -1,8 +1,9 @@ import unittest from mpyc.runtime import mpc from mpyc.random import (getrandbits, randrange, random_unit_vector, randint, - shuffle, random_permutation, random_derangement, + shuffle, np_shuffle, random_permutation, random_derangement, choice, choices, sample, random, uniform) +from mpyc.numpy import np class Arithmetic(unittest.TestCase): @@ -77,6 +78,29 @@ def test_secint(self): self.assertLessEqual(max(x) // 1000, 1009) self.assertEqual(sum(x) % 1000, 0) + @unittest.skipIf(not np, 'NumPy not available or inside MPyC disabled') + def test_np_shuffle(self): + secint = mpc.SecInt() + x = secint.array(np.arange(8)) + np_shuffle(x) + x = mpc.run(mpc.output(x)) + self.assertSetEqual(set(x), set(np.arange(8))) + + x_init = np.arange(8).reshape(2,4) + x = secint.array(x_init) + np_shuffle(x) + x = mpc.run(mpc.output(x)) + self.assertIn(set(x[0,:]), [set(x_init[i,:]) for i in range(x_init.shape[0])]) + + x = secint.array(x_init) + np_shuffle(x, axis=1) + x = mpc.run(mpc.output(x)) + self.assertIn(set(x[:,0]), [set(x_init[:,j]) for j in range(x_init.shape[1])]) + + x = secint.array(x_init) + self.assertRaises(ValueError, np_shuffle, x, 3) + + def test_secfxp(self): secfxp = mpc.SecFxp() a = getrandbits(secfxp, 10) From 465841b366baca8ad96d07071a7fa8c8c5174aeb Mon Sep 17 00:00:00 2001 From: DAMIE Marc Date: Mon, 24 Apr 2023 11:49:44 +0200 Subject: [PATCH 3/6] Minor fixes in np_shuflle --- mpyc/random.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/mpyc/random.py b/mpyc/random.py index 4f962943..f39b2f29 100644 --- a/mpyc/random.py +++ b/mpyc/random.py @@ -354,7 +354,7 @@ async def np_random_unit_vector(sectype, n): u = v elif await runtime.output(u[0] * x[i]): # TODO: mul_public # restart, keeping unused secret random bits x[:i] - x = runtime.np_hstack((x[:i], np_random_unit_vector(sectype, k - i))) + x = runtime.np_hstack((x[:i], runtime.np_random_bits(sectype, k - i))) i = k - 1 u = runtime.np_fromlist([x[i], 1 - x[i]]) else: @@ -370,16 +370,20 @@ def np_shuffle(a, axis=None): Given array x may contain public or secret elements. """ sectype = type(a).sectype + + if len(a.shape) > 2: + raise ValueError("Can only shuffle 1D and 2D arrays") + if axis is None: axis = 0 - if axis >= len(a.shape) or axis <0: + if axis not in (0,1,-1): raise ValueError("Invalid axis") x = runtime.np_copy(a) if axis != 0: - x = runtime.np_swapaxes(x, 0, axis) + x = runtime.np_transpose(x) n = x.shape[0] @@ -392,9 +396,9 @@ def np_shuffle(a, axis=None): else: d = u * (x[i] - x_u) x = runtime.np_hstack((x[:i, ...], runtime.np_add(x[i:, ...], d))) - runtime.np_update(x, i, x_u) + x = runtime.np_update(x, i, x_u) if axis != 0: - x = runtime.np_swapaxes(x, 0, axis) + x = runtime.np_transpose(x) runtime.np_update(a, range(a.shape[0]), x) From 57912d5a374527ca640ade3d51ec123af46aabc4 Mon Sep 17 00:00:00 2001 From: DAMIE Marc Date: Mon, 24 Apr 2023 12:55:56 +0200 Subject: [PATCH 4/6] Fix the in-place processing of np_shuffle --- mpyc/random.py | 3 ++- tests/test_random.py | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/mpyc/random.py b/mpyc/random.py index f39b2f29..fc196ba9 100644 --- a/mpyc/random.py +++ b/mpyc/random.py @@ -364,7 +364,7 @@ async def np_random_unit_vector(sectype, n): return u -def np_shuffle(a, axis=None): +async def np_shuffle(a, axis=None): """Shuffle numpy-like array x secretly in-place, and return None. Given array x may contain public or secret elements. @@ -401,4 +401,5 @@ def np_shuffle(a, axis=None): if axis != 0: x = runtime.np_transpose(x) + x = await runtime.gather(x) runtime.np_update(a, range(a.shape[0]), x) diff --git a/tests/test_random.py b/tests/test_random.py index 3941342b..f0b29c3e 100644 --- a/tests/test_random.py +++ b/tests/test_random.py @@ -82,23 +82,24 @@ def test_secint(self): def test_np_shuffle(self): secint = mpc.SecInt() x = secint.array(np.arange(8)) - np_shuffle(x) + mpc.run(np_shuffle(x)) x = mpc.run(mpc.output(x)) self.assertSetEqual(set(x), set(np.arange(8))) x_init = np.arange(8).reshape(2,4) x = secint.array(x_init) - np_shuffle(x) + mpc.run(np_shuffle(x)) x = mpc.run(mpc.output(x)) self.assertIn(set(x[0,:]), [set(x_init[i,:]) for i in range(x_init.shape[0])]) x = secint.array(x_init) - np_shuffle(x, axis=1) + mpc.run(np_shuffle(x, axis=1)) x = mpc.run(mpc.output(x)) self.assertIn(set(x[:,0]), [set(x_init[:,j]) for j in range(x_init.shape[1])]) x = secint.array(x_init) - self.assertRaises(ValueError, np_shuffle, x, 3) + with self.assertRaises(ValueError): + mpc.run(np_shuffle(x, 3)) def test_secfxp(self): From 1be00f0b40e2b17f085d70aad3dc268251106587 Mon Sep 17 00:00:00 2001 From: DAMIE Marc Date: Mon, 24 Apr 2023 15:53:53 +0200 Subject: [PATCH 5/6] Added unit test to improve coverage --- mpyc/random.py | 5 +---- tests/test_random.py | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/mpyc/random.py b/mpyc/random.py index fc196ba9..7592da06 100644 --- a/mpyc/random.py +++ b/mpyc/random.py @@ -332,10 +332,7 @@ async def np_random_unit_vector(sectype, n): with c a small constant, c < 3. """ - if issubclass(sectype, runtime.SecureObject): - await runtime.returnType((sectype.array, True, (n,))) - else: - await runtime.returnType(asyncoro.Future) + await runtime.returnType((sectype.array, True, (n,))) if n == 1: return runtime.np_fromlist([sectype(1)]) diff --git a/tests/test_random.py b/tests/test_random.py index f0b29c3e..ee42784d 100644 --- a/tests/test_random.py +++ b/tests/test_random.py @@ -1,6 +1,6 @@ import unittest from mpyc.runtime import mpc -from mpyc.random import (getrandbits, randrange, random_unit_vector, randint, +from mpyc.random import (getrandbits, randrange, random_unit_vector, np_random_unit_vector, randint, shuffle, np_shuffle, random_permutation, random_derangement, choice, choices, sample, random, uniform) from mpyc.numpy import np @@ -20,6 +20,8 @@ def test_secint(self): x = mpc.run(mpc.output(random_unit_vector(secint, 4))) self.assertEqual(sum(x), 1) + x = mpc.run(mpc.output(np_random_unit_vector(secint, 4))) + self.assertEqual(sum(x), 1) a = mpc.run(mpc.output(randrange(secint, 37))) # French roulette self.assertGreaterEqual(a, 0) @@ -86,6 +88,11 @@ def test_np_shuffle(self): x = mpc.run(mpc.output(x)) self.assertSetEqual(set(x), set(np.arange(8))) + x = secint.array(np.array([np.arange(8)])) + mpc.run(np_shuffle(x)) + x = mpc.run(mpc.output(x)) + self.assertTrue((x == np.array([np.arange(8)])).all()) + x_init = np.arange(8).reshape(2,4) x = secint.array(x_init) mpc.run(np_shuffle(x)) @@ -101,6 +108,10 @@ def test_np_shuffle(self): with self.assertRaises(ValueError): mpc.run(np_shuffle(x, 3)) + x = secint.array(np.ones((8,8,8))) + with self.assertRaises(ValueError): + mpc.run(np_shuffle(x)) + def test_secfxp(self): secfxp = mpc.SecFxp() @@ -112,6 +123,8 @@ def test_secfxp(self): x = mpc.run(mpc.output(random_unit_vector(secfxp, 3))) self.assertEqual(int(sum(x)), 1) + x = mpc.run(mpc.output(np_random_unit_vector(secfxp, 3))) + self.assertEqual(int(sum(x)), 1) x = mpc.run(mpc.output(random_permutation(secfxp, range(1, 9)))) self.assertSetEqual(set(map(int, x)), set(range(1, 9))) @@ -159,6 +172,8 @@ def test_secfld(self): self.assertIn(a, [0, 1, 2]) x = mpc.run(mpc.output(random_unit_vector(secfld, 2))) self.assertEqual(int(sum(x)), 1) + x = mpc.run(mpc.output(np_random_unit_vector(secfld, 2))) + self.assertEqual(int(sum(x)), 1) x = mpc.run(mpc.output(random_permutation(secfld, range(1, 9)))) self.assertSetEqual(set(map(int, x)), set(range(1, 9))) x = mpc.run(mpc.output(random_derangement(secfld, range(2)))) @@ -176,6 +191,8 @@ def test_secfld(self): self.assertIn(a, [-1, 0, 1]) x = mpc.run(mpc.output(random_unit_vector(secfld, 1))) self.assertEqual(int(sum(x)), 1) + x = mpc.run(mpc.output(np_random_unit_vector(secfld, 1))) + self.assertEqual(int(sum(x)), 1) x = mpc.run(mpc.output(random_permutation(secfld, range(1, 9)))) self.assertSetEqual(set(map(int, x)), set(range(1, 9))) x = mpc.run(mpc.output(random_derangement(secfld, range(2)))) From f5d673dbdd3f83dd8b61e17e3559fa1aa24506b4 Mon Sep 17 00:00:00 2001 From: DAMIE Marc Date: Mon, 24 Apr 2023 16:05:45 +0200 Subject: [PATCH 6/6] Fix unit tests when numpy is not available --- tests/test_random.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/tests/test_random.py b/tests/test_random.py index ee42784d..3a50abf2 100644 --- a/tests/test_random.py +++ b/tests/test_random.py @@ -20,8 +20,6 @@ def test_secint(self): x = mpc.run(mpc.output(random_unit_vector(secint, 4))) self.assertEqual(sum(x), 1) - x = mpc.run(mpc.output(np_random_unit_vector(secint, 4))) - self.assertEqual(sum(x), 1) a = mpc.run(mpc.output(randrange(secint, 37))) # French roulette self.assertGreaterEqual(a, 0) @@ -112,6 +110,23 @@ def test_np_shuffle(self): with self.assertRaises(ValueError): mpc.run(np_shuffle(x)) + @unittest.skipIf(not np, 'NumPy not available or inside MPyC disabled') + def test_np_random_unit_vector(self): + secint = mpc.SecInt() + x = mpc.run(mpc.output(np_random_unit_vector(secint, 4))) + self.assertEqual(sum(x), 1) + + secfxp = mpc.SecFxp() + x = mpc.run(mpc.output(np_random_unit_vector(secfxp, 3))) + self.assertEqual(int(sum(x)), 1) + + secfld = mpc.SecFld(256) + x = mpc.run(mpc.output(np_random_unit_vector(secfld, 2))) + self.assertEqual(int(sum(x)), 1) + + secfld = mpc.SecFld(257) + x = mpc.run(mpc.output(np_random_unit_vector(secfld, 1))) + self.assertEqual(int(sum(x)), 1) def test_secfxp(self): secfxp = mpc.SecFxp() @@ -123,8 +138,6 @@ def test_secfxp(self): x = mpc.run(mpc.output(random_unit_vector(secfxp, 3))) self.assertEqual(int(sum(x)), 1) - x = mpc.run(mpc.output(np_random_unit_vector(secfxp, 3))) - self.assertEqual(int(sum(x)), 1) x = mpc.run(mpc.output(random_permutation(secfxp, range(1, 9)))) self.assertSetEqual(set(map(int, x)), set(range(1, 9))) @@ -172,8 +185,6 @@ def test_secfld(self): self.assertIn(a, [0, 1, 2]) x = mpc.run(mpc.output(random_unit_vector(secfld, 2))) self.assertEqual(int(sum(x)), 1) - x = mpc.run(mpc.output(np_random_unit_vector(secfld, 2))) - self.assertEqual(int(sum(x)), 1) x = mpc.run(mpc.output(random_permutation(secfld, range(1, 9)))) self.assertSetEqual(set(map(int, x)), set(range(1, 9))) x = mpc.run(mpc.output(random_derangement(secfld, range(2)))) @@ -191,8 +202,6 @@ def test_secfld(self): self.assertIn(a, [-1, 0, 1]) x = mpc.run(mpc.output(random_unit_vector(secfld, 1))) self.assertEqual(int(sum(x)), 1) - x = mpc.run(mpc.output(np_random_unit_vector(secfld, 1))) - self.assertEqual(int(sum(x)), 1) x = mpc.run(mpc.output(random_permutation(secfld, range(1, 9)))) self.assertSetEqual(set(map(int, x)), set(range(1, 9))) x = mpc.run(mpc.output(random_derangement(secfld, range(2))))