improved cache hit

perpetual-ml · Jul 11, 2024 · 6f77401 · 6f77401
1 parent bc6a345
commit 6f77401
Show file tree

Hide file tree

Showing 8 changed files with 333 additions and 142 deletions.
diff --git a/benches/perpetual_benchmarks.rs b/benches/perpetual_benchmarks.rs
@@ -21,7 +21,7 @@ pub fn tree_benchmarks(c: &mut Criterion) {
         fs::read_to_string("resources/performance_100k_samp_seed0.csv").expect("Something went wrong reading the file");
     let y: Vec<f64> = file.lines().map(|x| x.parse::<f64>().unwrap()).collect();
     let yhat = vec![0.5; y.len()];
-    let (g, h) = LogLoss::calc_grad_hess(&y, &yhat, None, None);
+    let (mut g, mut h) = LogLoss::calc_grad_hess(&y, &yhat, None, None);
     let loss = LogLoss::calc_loss(&y, &yhat, None, None);
 
     let v: Vec<f32> = vec![10.; 300000];
@@ -56,8 +56,8 @@ pub fn tree_benchmarks(c: &mut Criterion) {
         data.index.to_owned(),
         &col_index,
         &bindata.cuts,
-        &g,
-        h.as_deref(),
+        &mut g,
+        h.as_deref_mut(),
         &splitter,
         true,
         Some(f32::MAX),
@@ -82,8 +82,8 @@ pub fn tree_benchmarks(c: &mut Criterion) {
                 black_box(data.index.to_owned()),
                 black_box(&col_index),
                 black_box(&bindata.cuts),
-                black_box(&g),
-                black_box(h.as_deref()),
+                black_box(&mut g),
+                black_box(h.as_deref_mut()),
                 black_box(&splitter),
                 black_box(false),
                 Some(f32::MAX),
@@ -108,8 +108,8 @@ pub fn tree_benchmarks(c: &mut Criterion) {
                 black_box(data.index.to_owned()),
                 black_box(&[1, 3, 4]),
                 black_box(&bindata.cuts),
-                black_box(&g),
-                black_box(h.as_deref()),
+                black_box(&mut g),
+                black_box(h.as_deref_mut()),
                 black_box(&splitter),
                 black_box(false),
                 Some(f32::MAX),

diff --git a/python-package/examples/performance_benchmark.ipynb b/python-package/examples/performance_benchmark.ipynb
@@ -174,7 +174,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "[I 2024-07-10 12:37:35,963] A new study created in memory with name: no-name-4c6a881c-9102-44d9-b018-3af8d37cb2ae\n"
+      "[I 2024-07-11 01:26:49,412] A new study created in memory with name: no-name-4a28ade5-6c54-459c-aaa7-2f3e0bd9c040\n"
      ]
     }
    ],
@@ -192,15 +192,15 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "[I 2024-07-10 12:37:36,403] Trial 0 finished with value: 1.0644386016870766 and parameters: {'learning_rate': 0.4073657656436648, 'min_split_gain': 0.0019204079494910193, 'reg_alpha': 0.685655809011563, 'reg_lambda': 0.019448941142879615, 'colsample_bytree': 0.7581830596778167, 'subsample': 0.3728715964643011, 'subsample_freq': 10, 'max_depth': 3, 'num_leaves': 260, 'min_child_samples': 44}. Best is trial 0 with value: 1.0644386016870766.\n"
+      "[I 2024-07-11 01:26:50,073] Trial 0 finished with value: 1.0644386016870766 and parameters: {'learning_rate': 0.4073657656436648, 'min_split_gain': 0.0019204079494910193, 'reg_alpha': 0.685655809011563, 'reg_lambda': 0.019448941142879615, 'colsample_bytree': 0.7581830596778167, 'subsample': 0.3728715964643011, 'subsample_freq': 10, 'max_depth': 3, 'num_leaves': 260, 'min_child_samples': 44}. Best is trial 0 with value: 1.0644386016870766.\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "CPU times: total: 422 ms\n",
-      "Wall time: 404 ms\n"
+      "CPU times: total: 469 ms\n",
+      "Wall time: 617 ms\n"
      ]
     }
    ],
@@ -410,14 +410,14 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "CPU times: total: 21.6 s\n",
-      "Wall time: 20.7 s\n"
+      "CPU times: total: 8.58 s\n",
+      "Wall time: 8.52 s\n"
      ]
     },
     {
      "data": {
       "text/plain": [
-       "<perpetual.PerpetualBooster at 0x1163632a260>"
+       "<perpetual.PerpetualBooster at 0x271374d9c90>"
       ]
      },
      "execution_count": 20,
@@ -427,7 +427,7 @@
    ],
    "source": [
     "%%time\n",
-    "model.fit(X_train, y_train, budget=1.5)"
+    "model.fit(X_train, y_train, budget=1.0)"
    ]
   },
   {
@@ -439,7 +439,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Test mse: 0.192352\n"
+      "Test mse: 0.198443\n"
      ]
     }
    ],
@@ -459,7 +459,7 @@
     {
      "data": {
       "text/plain": [
-       "244"
+       "106"
       ]
      },
      "execution_count": 22,
@@ -483,20 +483,20 @@
     "}\n",
     "</style>\n",
     "\n",
-    "| Perpetual budget | Seed | Perpetual mse | Perpetual cpu time |\n",
-    "| ---------------- | ---- | ------------- | ------------------ |\n",
-    "| 1.0              | 0    | 0.187273      | 9.23               |\n",
-    "| 1.0              | 1    | 0.189911      | 10.5               |\n",
-    "| 1.0              | 2    | 0.194937      | 11.0               |\n",
-    "| 1.0              | 3    | 0.182932      | 9.77               |\n",
-    "| 1.0              | 4    | 0.198443      | 9.88               |\n",
-    "| 1.0              | avg  | 0.190699      | 10.1               |\n",
-    "| 1.5              | 0    | 0.185843      | 28.6               |\n",
-    "| 1.5              | 1    | 0.188146      | 26.8               |\n",
-    "| 1.5              | 2    | 0.190484      | 26.6               |\n",
-    "| 1.5              | 3    | 0.178708      | 25.1               |\n",
-    "| 1.5              | 4    | 0.192352      | 21.6               |\n",
-    "| 1.5              | avg  | 0.187107      | 25.7               |"
+    "| Perpetual budget | Seed | Perpetual mse | Perpetual cpu time | cpu time improved |\n",
+    "| ---------------- | ---- | ------------- | ------------------ | ----------------- |\n",
+    "| 1.0              | 0    | 0.187273      | 9.23               | 9.28              |\n",
+    "| 1.0              | 1    | 0.189911      | 10.5               | 9.69              |\n",
+    "| 1.0              | 2    | 0.194937      | 11.0               | 11.0              |\n",
+    "| 1.0              | 3    | 0.182932      | 9.77               | 10.5              |\n",
+    "| 1.0              | 4    | 0.198443      | 9.88               | 8.58              |\n",
+    "| 1.0              | avg  | 0.190699      | 10.1               | 9.81              |\n",
+    "| 1.5              | 0    | 0.185843      | 28.6               | 27.2              |\n",
+    "| 1.5              | 1    | 0.188146      | 26.8               | 25.5              |\n",
+    "| 1.5              | 2    | 0.190484      | 26.6               | 25.2              |\n",
+    "| 1.5              | 3    | 0.178708      | 25.1               | 23.1              |\n",
+    "| 1.5              | 4    | 0.192352      | 21.6               | 20.8              |\n",
+    "| 1.5              | avg  | 0.187107      | 25.7               | 24.4              |"
    ]
   },
   {

diff --git a/src/booster.rs b/src/booster.rs
@@ -324,7 +324,6 @@ impl PerpetualBooster {
             self.base_score = calc_init_callables(&self.objective)(y, sample_weight, alpha);
             yhat = vec![self.base_score; y.len()];
         } else {
-            // self.lumber(data, y, sample_weight, alpha)?;
             yhat = self.predict(data, self.parallel, None);
         }
 
@@ -385,8 +384,8 @@ impl PerpetualBooster {
                 data.index.to_owned(),
                 &col_index,
                 &binned_data.cuts,
-                &grad,
-                hess.as_deref(),
+                &mut grad,
+                hess.as_deref_mut(),
                 splitter,
                 self.parallel,
                 tld,

diff --git a/src/histogram.rs b/src/histogram.rs
@@ -223,6 +223,8 @@ impl HistogramMatrix {
     #[allow(clippy::too_many_arguments)]
     pub fn update(
         &mut self,
+        start: usize,
+        stop: usize,
         data: &Matrix<u16>,
         cuts: &JaggedMatrix<f64>,
         grad: &[f32],
@@ -242,24 +244,19 @@ impl HistogramMatrix {
         let (sorted_grad, sorted_hess) = match hess {
             Some(hess) => {
                 if !sort {
-                    (grad.to_vec(), Some(hess.to_vec()))
+                    (grad, Some(hess))
                 } else {
-                    let mut n_grad = Vec::with_capacity(index.len());
-                    let mut n_hess = Vec::with_capacity(index.len());
-                    for i in index {
-                        let i_ = *i;
-                        n_grad.push(grad[i_]);
-                        n_hess.push(hess[i_]);
-                    }
-                    (n_grad, Some(n_hess))
+                    let g = &grad[start..stop];
+                    let h = &hess[start..stop];
+                    (g, Some(h))
                 }
             }
             None => {
                 if !sort {
-                    (grad.to_vec(), None::<Vec<f32>>)
+                    (grad, None)
                 } else {
-                    let n_grad = index.iter().map(|i| grad[*i]).collect::<Vec<f32>>();
-                    (n_grad, None::<Vec<f32>>)
+                    let g = &grad[start..stop];
+                    (g, None)
                 }
             }
         };
@@ -271,7 +268,7 @@ impl HistogramMatrix {
                 cuts.get_col(*col),
                 &sorted_grad,
                 sorted_hess.as_deref(),
-                index,
+                &index[start..stop],
             );
         });
     }
@@ -454,7 +451,18 @@ mod tests {
         let mut hist_init = HistogramMatrix::empty(&bdata, &b.cuts, &col_index, false, false);
 
         let hist_col = 1;
-        hist_init.update(&bdata, &b.cuts, &g, h.as_deref(), &bdata.index, &col_index, true, false);
+        hist_init.update(
+            0,
+            bdata.index.len(),
+            &bdata,
+            &b.cuts,
+            &g,
+            h.as_deref(),
+            &bdata.index,
+            &col_index,
+            true,
+            false,
+        );
 
         let mut f = bdata.get_col(hist_col).to_owned();
 

diff --git a/src/partial_dependence.rs b/src/partial_dependence.rs
@@ -94,7 +94,7 @@ mod tests {
         let file = fs::read_to_string("resources/performance.csv").expect("Something went wrong reading the file");
         let y: Vec<f64> = file.lines().map(|x| x.parse::<f64>().unwrap()).collect();
         let yhat = vec![0.5; y.len()];
-        let (g, h) = LogLoss::calc_grad_hess(&y, &yhat, None, None);
+        let (mut g, mut h) = LogLoss::calc_grad_hess(&y, &yhat, None, None);
         let loss = LogLoss::calc_loss(&y, &yhat, None, None);
 
         let data = Matrix::new(&data_vec, 891, 5);
@@ -120,8 +120,8 @@ mod tests {
             data.index.to_owned(),
             &col_index,
             &b.cuts,
-            &g,
-            h.as_deref(),
+            &mut g,
+            h.as_deref_mut(),
             &splitter,
             true,
             Some(f32::MAX),