hugely optimized by scaling the bw image for each window size, and more timers

src/test.rs

@@ -26,7 +26,6 @@ fn main() {
     // println!("Haar: {:?}", haar);
 
 
-    let mut sw = Stopwatch::start_new();
 
     let frame = image::open("test.png");
     
@@ -47,12 +46,13 @@ fn main() {
             // _model.image = Some(nannou::image::DynamicImage::ImageLuma8(ib_bw));
             let i = ib.as_rgb8().unwrap().clone();
             let hm = Some(heatmap::Heatmap::new(heatmap::ColorMaps::Plasma));
+            let mut sw = Stopwatch::start_new();
             let image = haar.scan_image(i, &hm).unwrap().dynamic_img;
+            info!("Scanning for faces took {}ms", sw.elapsed_ms());
             image.save("test-output.png");
 
             // let hm = heatmap::Heatmap::new(heatmap::ColorMaps::NipySpectral);
             // let hm = heatmap::Heatmap::new(heatmap::ColorMaps::TraficLight);
-            info!("Scanning for faces took {}ms", sw.elapsed_ms());
             // sw.restart();
             // let hm = h;
             // let image_hm = hm.convert_image(image);

src/visualhaar.rs

@@ -56,7 +56,7 @@ impl HaarClassifierFeature{
         let mut score = 0.;
 
         for rect in &self.rects{
-            score += rect.compute_rect(image_window, scale, x, y, scan_window_size);
+            score += rect.compute_rect(image_window, x, y);
         }
 
         score
@@ -91,12 +91,21 @@ impl HaarClassifierFeatureRect{
         (x1, y1, x2, y2)
     }
 
+    fn get_coordinates(&self) -> (usize, usize, usize, usize) {
+        let x1 = self.tl_x as usize;
+        let y1 = self.tl_y as usize;
+        let x2 = x1 + self.width as usize;
+        let y2 = y1 + self.height as usize;
+        
+        (x1, y1, x2, y2)
+    }
+
     /// The feature sum is finally calculated by first summing all values of the pixels inside the rectangle and then multiplying it with the weight factor. Finally, those weighted sums are combined together to yield as a final feature value. Keep in mind that all the coordinates retrieved for a single feature are in relation to the window/model size and not the complete image which is processed.
-    fn compute_rect(&self, image_window: &nd::ArrayView2<u32>, scale: &f64, x: usize, y: usize, scan_window_size: usize) -> f64 {
+    fn compute_rect(&self, image_window: &nd::ArrayView2<u32>, x: usize, y: usize) -> f64 {
-        let (x1, y1, x2, y2) = self.get_coordinates_for_scale(scale);
+        let (x1, y1, x2, y2) = self.get_coordinates();
         
         let sum = (image_window[[y+y2,x+x2]] + image_window[[y+y1,x+x1]] - image_window[[y+y1, x+x2]] - image_window[[y+y2, x+x1]]) as f64;
-        let sum = (sum/(scale*scale)) * self.weight as f64; // normalise: when the window grows, all values of the integral image become bigger by a factor scale-squared
+        let sum = (sum) * self.weight as f64; // normalise: when the window grows, all values of the integral image become bigger by a factor scale-squared
         return sum;
     }
 
@@ -334,18 +343,22 @@ impl HaarClassifier {
 
     /// take an ImageBuffer and scan it for faces.
     pub fn scan_image(&self, frame: image::ImageBuffer<image::Rgb<u8>, Vec<u8>>, heatmap: &Option<heatmap::Heatmap>) -> Result<Outcome, String> {
+
+
+        let sw = Stopwatch::start_new();
+
         let img_bw = image::imageops::grayscale(&frame);
         // let mut output_image = image::GrayImage::new(frame.width(), frame.height());
 
-        let integral = Self::integral_image(&img_bw);
+        // let integral = Self::integral_image(&img_bw);
 
         let mut output_frame: nd::Array2<i16> = nd::Array::zeros((
-            img_bw.dimensions().1 as usize,
+            img_bw.height() as usize,
-            img_bw.dimensions().0 as usize,
+            img_bw.width() as usize,
         ));
 
-        let mut integral_view = integral.view();
+        // let mut integral_view = integral.view();
-        let mut output_draw_frame = output_frame.view_mut();
+        // let mut output_draw_frame = output_frame.view_mut();
 
         // info!("Frame: {:?} {:?}", integral[[0,0]], integral[[integral.dim().0-1,integral.dim().1-1]]);
 
@@ -358,35 +371,75 @@ impl HaarClassifier {
         let mut window_size: usize = min_size.clone() as usize;
         let mut count_faces = 0;
         let mut count_not_faces = 0;
+
+        
+        info!("preprocessing: {:?}ms", sw.elapsed_ms());
+
+        let mut loop_time: i64 = 0;
+        
         while window_size < max_window_size {
 
             let sw = Stopwatch::start_new();
             let scale = (window_size-1) as f64 / self.width as f64;
+
+            let img_bw_scaled = image::imageops::resize(
+                &img_bw, 
+                (img_bw.width() as f64 / scale + 1.) as u32, 
+                (img_bw.height() as f64 / scale + 1.) as u32, 
+                image::imageops::FilterType::Nearest
+            );
+
+            let integral = Self::integral_image(&img_bw_scaled);
+
+            let mut scaled_output_frame: nd::Array2<i16> = nd::Array::zeros((
+                img_bw_scaled.dimensions().1 as usize,
+                img_bw_scaled.dimensions().0 as usize,
+            ));
+
+            let integral_view = integral.view();
+            let mut scaled_output_draw_frame = scaled_output_frame.view_mut();
+
             // to calculate a rect, we would need a -1 row, if we ignore that precision and add one at the end: (eg required when an item has width 20 (== feature width)) 
             let scan_window_size = window_size + 1;
-            info!("Window size: {:?} {:?}", window_size, scale);
+            info!("Window size: {:?} {:?} {:?}", window_size, scale, scaled_output_draw_frame.dim());
 
             // step by scale.ceil() as this is 1px in the model's size. (small is probably unnecesarily fine-grained)
-            for x in (0..(img_bw.dimensions().0 as usize - scan_window_size)).step_by((scale * 1.0).ceil() as usize)  {
+            for x in (0..(img_bw_scaled.width() as usize - self.width as usize))  {
-                for y in (0..(img_bw.dimensions().1 as usize - scan_window_size)).step_by((scale * 1.0).ceil() as usize) {
+                for y in (0..(img_bw_scaled.height() as usize - self.height as usize)) {
                     // let window = integral.slice(s![y..y+scan_window_size, x..x+scan_window_size]);
                     // let mut output_window = output_frame.slice_mut(s![y..y+scan_window_size, x..x+scan_window_size]);
-                    if self.scan_window(integral_view, scale, &mut output_draw_frame, x, y, window_size) {
+                    if self.scan_window(integral_view, 1., &mut scaled_output_draw_frame, x, y, window_size) {
                         count_faces += 1;
                     } else {
                         count_not_faces += 1;
                     }
-                    // break;
                 }
-                // break;
             }
 
-            info!("\ttook: {:?}ms", sw.elapsed_ms());
+            for x in 0..img_bw.width() {
+                for  y in 0..img_bw.height() {
+                    let src_x = (x as f64 / scale) as usize;
+                    let src_y = (y as f64 / scale) as usize;
+                    
+                    let weight = scaled_output_draw_frame[[src_y, src_x]];
+                    // let weight = img_bw_scaled.get_pixel(src_x, src_y).0.first().unwrap().clone() as i16;
+                    // info!("Pixel: {:?} {:?} {:?} {:?} {:?} {:?} {:?}", x, y, output_frame.dim(), scale, src_x, src_y, img_bw_scaled.dimensions());
+                    output_frame[[y as usize, x as usize]] += weight;
+                }
+            }
+
+            let elapsed = sw.elapsed_ms();
+            info!("\ttook: {:?}ms", elapsed);
             // break;
+            loop_time += elapsed;
 
             window_size = (window_size as f32 * 1.2) as usize; // TODO make grow-factor variable (now 1.2)
+
         }
 
+        info!("Looping took: {:?}", loop_time);
+        let sw = Stopwatch::start_new();
+
         // let mut test_window = output_frame.slice_mut(s![10..20,40..50]);
         // test_window += 10.;
 
@@ -418,10 +471,18 @@ impl HaarClassifier {
         
         // let dynamic = image::DynamicImage::ImageLuma8(img_bw);
         let dynamic = image::DynamicImage::ImageLuma8(final_img);
+
+
+        info!("postprocessing: {:?}ms", sw.elapsed_ms());
+
         let dynamic = match heatmap {
             Some(hm) => {
                 // TODO remove intermediate DynamicImage conversin
-                image::DynamicImage::ImageRgb8(hm.convert_image(dynamic))
+                
+                let sw = Stopwatch::start_new();
+                let i = image::DynamicImage::ImageRgb8(hm.convert_image(dynamic));
+                info!("heatmap: {:?}ms", sw.elapsed_ms());
+                i
             }
             None => {
                 // no changes needed