Fix slow push (#1392)

To determine which files are part of a filtered subset, regardless of their potentially changed path a trick is used: The filter is applied and then the inverted filter is applied to the result. The implementation of that trick was to simply chain the filter with the inverted one. However this caused flatten() to create a hughe intermediate representation (something like a cartesian product) that then has to be simplified down again by optimize()/simplify(). While that worked, it was very computationally expensive making push extremly slow in some cases. Now instead of chaining apply() is called twice, avoiding the huge intermediate representation and thus expensive optimization. Change: fix-slow-push
josh-project · Sep 24, 2024 · 7bc2eec · 7bc2eec
1 parent 7dffff0
commit 7bc2eec
Show file tree

Hide file tree

Showing 9 changed files with 18 additions and 69 deletions.
diff --git a/josh-core/src/filter/mod.rs b/josh-core/src/filter/mod.rs
@@ -1105,11 +1105,8 @@ pub fn unapply<'a>(
     parent_tree: git2::Tree<'a>,
 ) -> JoshResult<git2::Tree<'a>> {
     if let Ok(inverted) = invert(filter) {
-        let matching = apply(
-            transaction,
-            chain(invert(inverted)?, inverted),
-            parent_tree.clone(),
-        )?;
+        let filtered = apply(transaction, invert(inverted)?, parent_tree.clone())?;
+        let matching = apply(transaction, inverted, filtered)?;
         let stripped = tree::subtract(transaction, parent_tree.id(), matching.id())?;
         let new_tree = apply(transaction, inverted, tree)?;
 
@@ -1166,11 +1163,8 @@ fn unapply_workspace<'a>(
             let wsj_file = chain(root, wsj_file);
             let filter = compose(wsj_file, compose(workspace, root));
             let original_filter = compose(wsj_file, compose(original_workspace, root));
-            let matching = apply(
-                transaction,
-                chain(original_filter, invert(original_filter)?),
-                parent_tree.clone(),
-            )?;
+            let filtered = apply(transaction, original_filter, parent_tree.clone())?;
+            let matching = apply(transaction, invert(original_filter)?, filtered)?;
             let stripped = tree::subtract(transaction, parent_tree.id(), matching.id())?;
             let new_tree = apply(transaction, invert(filter)?, tree)?;
 

diff --git a/tests/proxy/workspace.t b/tests/proxy/workspace.t
@@ -335,8 +335,6 @@
       |   |   `-- f4045367114a7584eefa64b95bb69d7f840aef
       |   |-- 64
       |   |   `-- d1f8d32b274d8c1eeb69891931f52b6ade9417
-      |   |-- 66
-      |   |   `-- b81c71c0ad10acdb2b4df3b04eef8abd79e64b
       |   |-- 69
       |   |   `-- d753308940fb9a6c60bf04d0b1c7421fd91aba
       |   |-- 77
@@ -365,8 +363,6 @@
       |   |   `-- 76696a87dd29d841caa7f0d48e2f4e5359d369
       |   |-- bc
       |   |   `-- 665856e841c4ae4a956483dc57b2ea4cc20116
-      |   |-- bd
-      |   |   `-- 495daf53fe6fd641cc91e8208674050f602955
       |   |-- be
       |   |   |-- 06ec33a91b8a108bd135b5a42a84292b035d6b
       |   |   `-- 1afe91719c4fb58e09207874905f09858cfb66
@@ -404,6 +400,6 @@
           |-- namespaces
           `-- tags
 
-  96 directories, 84 files
+  94 directories, 82 files
 
 $ cat ${TESTTMP}/josh-proxy.out
diff --git a/tests/proxy/workspace_create.t b/tests/proxy/workspace_create.t
@@ -611,8 +611,7 @@ Note that ws/d/ is now present in the ws
       |   |-- 40
       |   |   `-- c389b6b248e13f3cb88dcd79467d7396a4489e
       |   |-- 43
-      |   |   |-- 52611a9e7c56dfdfeadec043ced6d6ef7a5c33
-      |   |   `-- c475ca897b62fd739409aee5db69b0c480aa0d
+      |   |   `-- 52611a9e7c56dfdfeadec043ced6d6ef7a5c33
       |   |-- 46
       |   |   `-- bc1eabe4b2029b9fcb661f0d447d8389d17337
       |   |-- 47
@@ -622,8 +621,6 @@ Note that ws/d/ is now present in the ws
       |   |   `-- 825dc642cb6eb9a060e54bf8d69288fbee4904
       |   |-- 4c
       |   |   `-- a9fbf65ee12663ac24db4be4cab10e53d6d6e7
-      |   |-- 4d
-      |   |   `-- dbf4f812dbd547947a2511b01985fb7a460eae
       |   |-- 5b
       |   |   `-- 560252b0c3c6abc5e0327596a49efb15e494cb
       |   |-- 5e
@@ -634,8 +631,6 @@ Note that ws/d/ is now present in the ws
       |   |-- 64
       |   |   |-- 6fd2c5bfe156d57ba03f62f2fe735ddbb74e22
       |   |   `-- d1f8d32b274d8c1eeb69891931f52b6ade9417
-      |   |-- 66
-      |   |   `-- b81c71c0ad10acdb2b4df3b04eef8abd79e64b
       |   |-- 6c
       |   |   `-- 68dd37602c8e2036362ab81b12829c4d6c0867
       |   |-- 6d
@@ -684,8 +679,6 @@ Note that ws/d/ is now present in the ws
       |   |   |-- 269dc50ffcdc1b87664a061926bf9a072a3456
       |   |   |-- a7760c60ac08ecdcac395823637989a4d681a6
       |   |   `-- c31372c5de4fb705ffdcbf5a4ec5c5103231d9
-      |   |-- a9
-      |   |   `-- 954d28dd1313fb2d8f20fc11a1106376c2ffae
       |   |-- b1
       |   |   `-- c3c9b731fea98c82ce8e9286e213fcb033f70d
       |   |-- b2
@@ -700,8 +693,6 @@ Note that ws/d/ is now present in the ws
       |   |   `-- f9dcf1394d5152de67b115f55f25e4dc0a2398
       |   |-- bc
       |   |   `-- 665856e841c4ae4a956483dc57b2ea4cc20116
-      |   |-- bd
-      |   |   `-- 495daf53fe6fd641cc91e8208674050f602955
       |   |-- bf
       |   |   `-- 7841c0db704465dda4b387d5da09694647d188
       |   |-- c1
@@ -755,6 +746,6 @@ Note that ws/d/ is now present in the ws
           |-- namespaces
           `-- tags
 
-  154 directories, 160 files
+  150 directories, 155 files
 
 $ cat ${TESTTMP}/josh-proxy.out
diff --git a/tests/proxy/workspace_edit_commit.t b/tests/proxy/workspace_edit_commit.t
@@ -295,6 +295,8 @@
       |   |   `-- cff5ef0fce401a1a33c2ac2713d6356cbc1b15
       |   |-- 1b
       |   |   `-- 46698f32d1d1db1eaeb34f8c9037778d65f3a9
+      |   |-- 1f
+      |   |   `-- 8abd6f6494fc2f715b3c1a39ac2e01ba806938
       |   |-- 20
       |   |   `-- 31777de79cd7c74834915674377e96d6864cc9
       |   |-- 22
@@ -337,8 +339,6 @@
       |   |   `-- 7ff045529989036cbd11bc32b2eaf5a8311aea
       |   |-- 60
       |   |   `-- 5066c26f66fca5a424aa32bd042ae71c7c8705
-      |   |-- 66
-      |   |   `-- b81c71c0ad10acdb2b4df3b04eef8abd79e64b
       |   |-- 6a
       |   |   `-- 80a5b3af9023d11cb7f37bc1f80d1d1805bfdb
       |   |-- 6c
@@ -387,7 +387,6 @@
       |   |   |-- c8440fe2cd36638ddb6b3505c1e8f2202f6191
       |   |   `-- cfe79e25ecd337b379e7ec06d7939dbcb2f6a0
       |   |-- bd
-      |   |   |-- 495daf53fe6fd641cc91e8208674050f602955
       |   |   `-- 56f16bf42ff74e68cfb7a59869c81920b02b87
       |   |-- be
       |   |   `-- c9383652a22b8a07acb86d5357a75f988286dc
@@ -433,4 +432,4 @@
           |-- namespaces
           `-- tags
 
-  129 directories, 124 files
+  129 directories, 123 files
diff --git a/tests/proxy/workspace_in_workspace.t b/tests/proxy/workspace_in_workspace.t
@@ -397,8 +397,6 @@
       |   |   `-- 5066c26f66fca5a424aa32bd042ae71c7c8705
       |   |-- 64
       |   |   `-- d1f8d32b274d8c1eeb69891931f52b6ade9417
-      |   |-- 66
-      |   |   `-- b81c71c0ad10acdb2b4df3b04eef8abd79e64b
       |   |-- 68
       |   |   `-- b1430cedd477c8117976bdb8aba3cee8b9aa9e
       |   |-- 6b
@@ -439,8 +437,6 @@
       |   |   `-- bd62ec41c785d12270e69b9d49f9babe62fcd6
       |   |-- bc
       |   |   `-- 665856e841c4ae4a956483dc57b2ea4cc20116
-      |   |-- bd
-      |   |   `-- 495daf53fe6fd641cc91e8208674050f602955
       |   |-- c2
       |   |   `-- d86319b61f31a7f4f1bc89b8ea4356b60c4658
       |   |-- c5
@@ -474,5 +470,5 @@
           |-- namespaces
           `-- tags
 
-  119 directories, 111 files
+  117 directories, 109 files
 
diff --git a/tests/proxy/workspace_modify.t b/tests/proxy/workspace_modify.t
@@ -616,8 +616,7 @@ Note that ws/d/ is now present in the ws
       |   |-- 40
       |   |   `-- c389b6b248e13f3cb88dcd79467d7396a4489e
       |   |-- 43
-      |   |   |-- 52611a9e7c56dfdfeadec043ced6d6ef7a5c33
-      |   |   `-- c475ca897b62fd739409aee5db69b0c480aa0d
+      |   |   `-- 52611a9e7c56dfdfeadec043ced6d6ef7a5c33
       |   |-- 44
       |   |   `-- 625a9b34b1c6747c29903c3e641a4b2e580673
       |   |-- 46
@@ -628,8 +627,6 @@ Note that ws/d/ is now present in the ws
       |   |   `-- 825dc642cb6eb9a060e54bf8d69288fbee4904
       |   |-- 4c
       |   |   `-- a9fbf65ee12663ac24db4be4cab10e53d6d6e7
-      |   |-- 4d
-      |   |   `-- dbf4f812dbd547947a2511b01985fb7a460eae
       |   |-- 59
       |   |   `-- 632d8d838ce9390679767c02c6bfe6c0d244a9
       |   |-- 5a
@@ -643,8 +640,6 @@ Note that ws/d/ is now present in the ws
       |   |-- 64
       |   |   |-- 6fd2c5bfe156d57ba03f62f2fe735ddbb74e22
       |   |   `-- d1f8d32b274d8c1eeb69891931f52b6ade9417
-      |   |-- 66
-      |   |   `-- b81c71c0ad10acdb2b4df3b04eef8abd79e64b
       |   |-- 6c
       |   |   `-- 68dd37602c8e2036362ab81b12829c4d6c0867
       |   |-- 6d
@@ -694,8 +689,6 @@ Note that ws/d/ is now present in the ws
       |   |   |-- 1e8a91058875f157ca1246bdc403b88e93cd94
       |   |   |-- 269dc50ffcdc1b87664a061926bf9a072a3456
       |   |   `-- c31372c5de4fb705ffdcbf5a4ec5c5103231d9
-      |   |-- a9
-      |   |   `-- 954d28dd1313fb2d8f20fc11a1106376c2ffae
       |   |-- ab
       |   |   `-- a295fbe181a47f04650542b7d5582fbd983b98
       |   |-- b1
@@ -709,8 +702,6 @@ Note that ws/d/ is now present in the ws
       |   |   `-- f9dcf1394d5152de67b115f55f25e4dc0a2398
       |   |-- bc
       |   |   `-- 665856e841c4ae4a956483dc57b2ea4cc20116
-      |   |-- bd
-      |   |   `-- 495daf53fe6fd641cc91e8208674050f602955
       |   |-- c1
       |   |   `-- 489fc8fd6ae9ac08c0168d7cabaf5645b922fa
       |   |-- c2
@@ -766,6 +757,6 @@ Note that ws/d/ is now present in the ws
           |-- namespaces
           `-- tags
 
-  163 directories, 163 files
+  159 directories, 158 files
 
 $ cat ${TESTTMP}/josh-proxy.out
diff --git a/tests/proxy/workspace_modify_chain.t b/tests/proxy/workspace_modify_chain.t
@@ -396,8 +396,6 @@
       |   |-- 64
       |   |   |-- 6fd2c5bfe156d57ba03f62f2fe735ddbb74e22
       |   |   `-- d1f8d32b274d8c1eeb69891931f52b6ade9417
-      |   |-- 66
-      |   |   `-- ba3cf39148efcb72db4e7b663e4f42247e2273
       |   |-- 67
       |   |   `-- 48a16f7d9b29607fcd4df93681361a6105a14a
       |   |-- 69
@@ -440,8 +438,6 @@
       |   |   `-- 0969c65843463b2d0e86fd4c6efcae33012332
       |   |-- b0
       |   |   `-- 3b5fbc9a12109cd3f5308929e4812b3c998da6
-      |   |-- b7
-      |   |   `-- dfc3aee5bd47bcfcf5a40b7964164a2b1e838b
       |   |-- b8
       |   |   `-- 54e8ec3db62174179b215404936a58f1bb6a79
       |   |-- b9
@@ -460,8 +456,7 @@
       |   |   |-- 0544cd5f702a9b3418205ec0425c6ae77f9e3e
       |   |   `-- 7c94c08f59302a2b1587a01d4fd1680d7378c9
       |   |-- d7
-      |   |   |-- 330ea337031af43ba1cf6982a873a40b9170ac
-      |   |   `-- 94016e24d97a10d550de2384527f80f9fecfc8
+      |   |   `-- 330ea337031af43ba1cf6982a873a40b9170ac
       |   |-- e1
       |   |   `-- 25e6d9f8f9acca5ffd25ee3c97d09748ad2a8b
       |   |-- e5
@@ -492,6 +487,6 @@
           |-- namespaces
           `-- tags
 
-  124 directories, 120 files
+  122 directories, 117 files
 
 $ cat ${TESTTMP}/josh-proxy.out | grep VIEW
diff --git a/tests/proxy/workspace_modify_chain_prefix_subtree.t b/tests/proxy/workspace_modify_chain_prefix_subtree.t
@@ -469,8 +469,7 @@ Note that ws/d/ is now present in the ws
       |   |-- 40
       |   |   `-- c389b6b248e13f3cb88dcd79467d7396a4489e
       |   |-- 43
-      |   |   |-- 52611a9e7c56dfdfeadec043ced6d6ef7a5c33
-      |   |   `-- c475ca897b62fd739409aee5db69b0c480aa0d
+      |   |   `-- 52611a9e7c56dfdfeadec043ced6d6ef7a5c33
       |   |-- 44
       |   |   |-- 625a9b34b1c6747c29903c3e641a4b2e580673
       |   |   `-- edc62d506b9805a3edfc74db15b1cc0bfc6871
@@ -482,8 +481,6 @@ Note that ws/d/ is now present in the ws
       |   |   `-- 825dc642cb6eb9a060e54bf8d69288fbee4904
       |   |-- 4c
       |   |   `-- a9fbf65ee12663ac24db4be4cab10e53d6d6e7
-      |   |-- 4d
-      |   |   `-- dbf4f812dbd547947a2511b01985fb7a460eae
       |   |-- 5a
       |   |   `-- 8a4b8c1452d54f1ca88454067369112809a4d2
       |   |-- 5b
@@ -493,8 +490,6 @@ Note that ws/d/ is now present in the ws
       |   |-- 64
       |   |   |-- 6fd2c5bfe156d57ba03f62f2fe735ddbb74e22
       |   |   `-- d1f8d32b274d8c1eeb69891931f52b6ade9417
-      |   |-- 66
-      |   |   `-- b81c71c0ad10acdb2b4df3b04eef8abd79e64b
       |   |-- 67
       |   |   `-- 12cb1b8c89e3b2272182f140c81aef3b718671
       |   |-- 6c
@@ -547,8 +542,6 @@ Note that ws/d/ is now present in the ws
       |   |   `-- c31372c5de4fb705ffdcbf5a4ec5c5103231d9
       |   |-- a4
       |   |   `-- b68220bdf7fb846eb9780f7846a2f4bf7cbcc3
-      |   |-- a9
-      |   |   `-- 954d28dd1313fb2d8f20fc11a1106376c2ffae
       |   |-- ab
       |   |   `-- a295fbe181a47f04650542b7d5582fbd983b98
       |   |-- b1
@@ -561,8 +554,6 @@ Note that ws/d/ is now present in the ws
       |   |   `-- f9dcf1394d5152de67b115f55f25e4dc0a2398
       |   |-- bc
       |   |   `-- 665856e841c4ae4a956483dc57b2ea4cc20116
-      |   |-- bd
-      |   |   `-- 495daf53fe6fd641cc91e8208674050f602955
       |   |-- c1
       |   |   `-- 489fc8fd6ae9ac08c0168d7cabaf5645b922fa
       |   |-- c2
@@ -606,6 +597,6 @@ Note that ws/d/ is now present in the ws
           |-- namespaces
           `-- tags
 
-  151 directories, 152 files
+  147 directories, 147 files
 
 $ cat ${TESTTMP}/josh-proxy.out | grep VIEW
diff --git a/tests/proxy/workspace_tags.t b/tests/proxy/workspace_tags.t
@@ -371,8 +371,6 @@
       |   |   `-- 5066c26f66fca5a424aa32bd042ae71c7c8705
       |   |-- 64
       |   |   `-- d1f8d32b274d8c1eeb69891931f52b6ade9417
-      |   |-- 66
-      |   |   `-- b81c71c0ad10acdb2b4df3b04eef8abd79e64b
       |   |-- 6b
       |   |   `-- e0d68b8e87001c8b91281636e21d6387010332
       |   |-- 78
@@ -408,8 +406,6 @@
       |   |   `-- bd62ec41c785d12270e69b9d49f9babe62fcd6
       |   |-- bc
       |   |   `-- 665856e841c4ae4a956483dc57b2ea4cc20116
-      |   |-- bd
-      |   |   `-- 495daf53fe6fd641cc91e8208674050f602955
       |   |-- c2
       |   |   `-- d86319b61f31a7f4f1bc89b8ea4356b60c4658
       |   |-- c5
@@ -440,6 +436,6 @@
           |-- namespaces
           `-- tags
 
-  109 directories, 101 files
+  107 directories, 99 files
 
 $ cat ${TESTTMP}/josh-proxy.out