Comment to use release build so that it becomes _way_ faster

README.md

@@ -0,0 +1,15 @@
+# Viola-Jones' Haarcascade visualisation
+
+This is an unusual visualisation of haarcascades. Often, only haarcascade features are drawn. This visualisation however, adds up all matching features in an image. Resulting in a sort of lense which shows the contrasts in the image that the algorithm picks up on most.
+
+* The visual_haarcascades binary ([main.rs](src/main.rs)) uses V4L to capture webcam input and renders to a canvas.
+* [test.rs](src/test.rs) binary is an ugly program that looks hard-coded for `haarcascade_frontalface_alt2.xml` and `test.png`. It renders the output to `test-output.png`.
+* The library is made for use with eg. Python through cffi. See eg. [this repo](https://git.rubenvandeven.com/r/face_recognition) 
+
+## build
+
+Most importantly
+
+```bash
+cargo build --lib --release
+```

src/lib.rs

@@ -43,6 +43,7 @@ pub extern "C" fn scan_image(haar: *mut visualhaar::HaarClassifier,
                             input: *const u8,
                             buffer: *mut u8,
                             length: usize,
+                            min_face_factor: usize,
                             debug: bool) {
     if haar.is_null() || input.is_null() || buffer.is_null() {
         return;
@@ -72,7 +73,7 @@ pub extern "C" fn scan_image(haar: *mut visualhaar::HaarClassifier,
         }
     }
 
-    let image = haar.scan_image(input_frame, &hm).unwrap().dynamic_img;
+    let image = haar.scan_image(input_frame, &hm,min_face_factor as u32).unwrap().dynamic_img;
     let rgb_img = image.to_rgb();
     // image.save("/home/ruben/Documents/Projecten/2020/rust/lena_orig-output-lib.png");
     info!("Scanning for faces took done");

src/test.rs

@@ -22,11 +22,10 @@ fn main() {
     
 
     warn!("test");
-    let haar = visualhaar::HaarClassifier::from_xml("haarcascade_frontalface_alt2.xml").unwrap();
+    let haar = visualhaar::HaarClassifier::from_xml("../haarcascade_frontalface_alt2.xml").unwrap();
     // println!("Haar: {:?}", haar);
 
 
-    let mut sw = Stopwatch::start_new();
 
     let frame = image::open("test.png");
     
@@ -47,12 +46,13 @@ fn main() {
             // _model.image = Some(nannou::image::DynamicImage::ImageLuma8(ib_bw));
             let i = ib.as_rgb8().unwrap().clone();
             let hm = Some(heatmap::Heatmap::new(heatmap::ColorMaps::Plasma));
-            let image = haar.scan_image(i, &hm).unwrap().dynamic_img;
+            let mut sw = Stopwatch::start_new();
+            let image = haar.scan_image(i, &hm, 4).unwrap().dynamic_img;
+            info!("Scanning for faces took {}ms", sw.elapsed_ms());
             image.save("test-output.png");
 
             // let hm = heatmap::Heatmap::new(heatmap::ColorMaps::NipySpectral);
             // let hm = heatmap::Heatmap::new(heatmap::ColorMaps::TraficLight);
-            info!("Scanning for faces took {}ms", sw.elapsed_ms());
             // sw.restart();
             // let hm = h;
             // let image_hm = hm.convert_image(image);

src/visualhaar.rs

@@ -52,19 +52,19 @@ pub struct HaarClassifierFeature {
 
 
 impl HaarClassifierFeature{
-    fn compute_feature(&self, image_window: &nd::ArrayView2<u32>, scale: &f64) -> f64 {
+    fn compute_feature(&self, image_window: &nd::ArrayView2<u32>, scale: &f64, x: usize, y: usize, scan_window_size: usize) -> f64 {
         let mut score = 0.;
 
         for rect in &self.rects{
-            score += rect.compute_rect(image_window, scale);
+            score += rect.compute_rect(image_window, x, y);
         }
 
         score
     }
 
-    fn draw(&self, draw_window: &mut  nd::ArrayViewMut2<i16>, scale: &f64) {
+    fn draw(&self, draw_window: &mut  nd::ArrayViewMut2<i16>, scale: &f64, x: usize, y: usize, scan_window_size: usize) {
         for rect in &self.rects{
-            rect.draw(draw_window, scale);
+            rect.draw(draw_window, scale, x, y, scan_window_size);
         }
     }
 }
@@ -91,24 +91,35 @@ impl HaarClassifierFeatureRect{
         (x1, y1, x2, y2)
     }
 
-    /// The feature sum is finally calculated by first summing all values of the pixels inside the rectangle and then multiplying it with the weight factor. Finally, those weighted sums are combined together to yield as a final feature value. Keep in mind that all the coordinates retrieved for a single feature are in relation to the window/model size and not the complete image which is processed.
+    fn get_coordinates(&self) -> (usize, usize, usize, usize) {
-    fn compute_rect(&self, image_window: &nd::ArrayView2<u32>, scale: &f64) -> f64 {
+        let x1 = self.tl_x as usize;
-        let (x1, y1, x2, y2) = self.get_coordinates_for_scale(scale);
+        let y1 = self.tl_y as usize;
+        let x2 = x1 + self.width as usize;
+        let y2 = y1 + self.height as usize;
         
-        let sum = (image_window[[y2,x2]] + image_window[[y1,x1]] - image_window[[y1, x2]] - image_window[[y2, x1]]) as f64;
+        (x1, y1, x2, y2)
-        let sum = (sum/(scale*scale)) * self.weight as f64; // normalise: when the window grows, all values of the integral image become bigger by a factor scale-squared
+    }
+
+    /// The feature sum is finally calculated by first summing all values of the pixels inside the rectangle and then multiplying it with the weight factor. Finally, those weighted sums are combined together to yield as a final feature value. Keep in mind that all the coordinates retrieved for a single feature are in relation to the window/model size and not the complete image which is processed.
+    fn compute_rect(&self, image_window: &nd::ArrayView2<u32>, x: usize, y: usize) -> f64 {
+        let (x1, y1, x2, y2) = self.get_coordinates();
+        
+        let sum = (image_window[[y+y2,x+x2]] + image_window[[y+y1,x+x1]] - image_window[[y+y1, x+x2]] - image_window[[y+y2, x+x1]]) as f64;
+        let sum = (sum) * self.weight as f64; // normalise: when the window grows, all values of the integral image become bigger by a factor scale-squared
         return sum;
     }
 
 
-    fn draw(&self, draw_window: &mut nd::ArrayViewMut2<i16>, scale: &f64) {
+    fn draw(&self, draw_window: &mut nd::ArrayViewMut2<i16>, scale: &f64, x: usize, y: usize, scan_window_size: usize) {
         let (x1, y1, x2, y2) = self.get_coordinates_for_scale(scale);
 
         // TODO how to speed this up?
         
         // info!("Draw {} {} {} {} ({:?}),", x1, y1, x2, y2,self);
-        let mut rect = draw_window.slice_mut(s![y1..y2, x1..x2]); // semi slow (initially 500ms)
+        let mut rect = draw_window.slice_mut(s![y+y1..y+y2, x+x1..x+x2]); // semi slow (initially 500ms)
+
         rect += self.weight; // super slow (initially 10.000 ms)
+        
         // info!("add")
         // for x in x1..x2{
         //     for y in y1..y2{
@@ -331,57 +342,105 @@ impl HaarClassifier {
     // }
 
     /// take an ImageBuffer and scan it for faces.
-    pub fn scan_image(&self, frame: image::ImageBuffer<image::Rgb<u8>, Vec<u8>>, heatmap: &Option<heatmap::Heatmap>) -> Result<Outcome, String> {
+    /// min_face_factor parameter gives starting size of scan window (frame height / factor). So higher number scans for smaller faces.
+    pub fn scan_image(&self, frame: image::ImageBuffer<image::Rgb<u8>, Vec<u8>>, heatmap: &Option<heatmap::Heatmap>, min_face_factor: u32) -> Result<Outcome, String> {
+
+
+        let sw = Stopwatch::start_new();
+
         let img_bw = image::imageops::grayscale(&frame);
         // let mut output_image = image::GrayImage::new(frame.width(), frame.height());
 
-        let integral = Self::integral_image(&img_bw);
+        // let integral = Self::integral_image(&img_bw);
 
         let mut output_frame: nd::Array2<i16> = nd::Array::zeros((
-            img_bw.dimensions().1 as usize,
+            img_bw.height() as usize,
-            img_bw.dimensions().0 as usize,
+            img_bw.width() as usize,
         ));
 
+        // let mut integral_view = integral.view();
+        // let mut output_draw_frame = output_frame.view_mut();
+
         // info!("Frame: {:?} {:?}", integral[[0,0]], integral[[integral.dim().0-1,integral.dim().1-1]]);
 
         // let rect = integral.slice(s![3..5, 2..4]);
 
         // let min_size = self.width;
-        let min_size = frame.height() / 3; // TODO: Make min face size (or factor) variable
+        let min_size = frame.height() / min_face_factor; // TODO: Make min face size (or factor) variable
         let max_window_size = std::cmp::min(img_bw.dimensions().0, img_bw.dimensions().1) as usize;
         
         let mut window_size: usize = min_size.clone() as usize;
         let mut count_faces = 0;
         let mut count_not_faces = 0;
+
+        
+        info!("preprocessing: {:?}ms", sw.elapsed_ms());
+
+        let mut loop_time: i64 = 0;
+        
         while window_size < max_window_size {
 
             let sw = Stopwatch::start_new();
             let scale = (window_size-1) as f64 / self.width as f64;
+
+            let img_bw_scaled = image::imageops::resize(
+                &img_bw, 
+                (img_bw.width() as f64 / scale + 1.) as u32, 
+                (img_bw.height() as f64 / scale + 1.) as u32, 
+                image::imageops::FilterType::CatmullRom
+            );
+
+            let integral = Self::integral_image(&img_bw_scaled);
+
+            let mut scaled_output_frame: nd::Array2<i16> = nd::Array::zeros((
+                img_bw_scaled.dimensions().1 as usize,
+                img_bw_scaled.dimensions().0 as usize,
+            ));
+
+            let integral_view = integral.view();
+            let mut scaled_output_draw_frame = scaled_output_frame.view_mut();
+
             // to calculate a rect, we would need a -1 row, if we ignore that precision and add one at the end: (eg required when an item has width 20 (== feature width)) 
             let scan_window_size = window_size + 1;
-            info!("Window size: {:?} {:?}", window_size, scale);
+            info!("Window size: {:?} {:?} {:?}", window_size, scale, scaled_output_draw_frame.dim());
 
             // step by scale.ceil() as this is 1px in the model's size. (small is probably unnecesarily fine-grained)
-            for x in (0..(img_bw.dimensions().0 as usize - scan_window_size)).step_by((scale * 1.0).ceil() as usize)  {
+            for x in (0..(img_bw_scaled.width() as usize - self.width as usize))  {
-                for y in (0..(img_bw.dimensions().1 as usize - scan_window_size)).step_by((scale * 1.0).ceil() as usize) {
+                for y in (0..(img_bw_scaled.height() as usize - self.height as usize)) {
-                    let window = integral.slice(s![y..y+scan_window_size, x..x+scan_window_size]);
+                    // let window = integral.slice(s![y..y+scan_window_size, x..x+scan_window_size]);
-                    let mut output_window = output_frame.slice_mut(s![y..y+scan_window_size, x..x+scan_window_size]);
+                    // let mut output_window = output_frame.slice_mut(s![y..y+scan_window_size, x..x+scan_window_size]);
-                    if self.scan_window(window, scale, &mut output_window) {
+                    if self.scan_window(integral_view, 1., &mut scaled_output_draw_frame, x, y, window_size) {
                         count_faces += 1;
                     } else {
                         count_not_faces += 1;
                     }
-                    // break;
                 }
-                // break;
             }
 
-            info!("\ttook: {:?}ms", sw.elapsed_ms());
+            for x in 0..img_bw.width() {
+                for  y in 0..img_bw.height() {
+                    let src_x = (x as f64 / scale) as usize;
+                    let src_y = (y as f64 / scale) as usize;
+                    
+                    let weight = scaled_output_draw_frame[[src_y, src_x]];
+                    // let weight = img_bw_scaled.get_pixel(src_x, src_y).0.first().unwrap().clone() as i16;
+                    // info!("Pixel: {:?} {:?} {:?} {:?} {:?} {:?} {:?}", x, y, output_frame.dim(), scale, src_x, src_y, img_bw_scaled.dimensions());
+                    output_frame[[y as usize, x as usize]] += weight;
+                }
+            }
+
+            let elapsed = sw.elapsed_ms();
+            info!("\ttook: {:?}ms", elapsed);
             // break;
+            loop_time += elapsed;
 
             window_size = (window_size as f32 * 1.2) as usize; // TODO make grow-factor variable (now 1.2)
+
         }
 
+        info!("Looping took: {:?}", loop_time);
+        let sw = Stopwatch::start_new();
+
         // let mut test_window = output_frame.slice_mut(s![10..20,40..50]);
         // test_window += 10.;
 
@@ -413,10 +472,18 @@ impl HaarClassifier {
         
         // let dynamic = image::DynamicImage::ImageLuma8(img_bw);
         let dynamic = image::DynamicImage::ImageLuma8(final_img);
+
+
+        info!("postprocessing: {:?}ms", sw.elapsed_ms());
+
         let dynamic = match heatmap {
             Some(hm) => {
                 // TODO remove intermediate DynamicImage conversin
-                image::DynamicImage::ImageRgb8(hm.convert_image(dynamic))
+                
+                let sw = Stopwatch::start_new();
+                let i = image::DynamicImage::ImageRgb8(hm.convert_image(dynamic));
+                info!("heatmap: {:?}ms", sw.elapsed_ms());
+                i
             }
             None => {
                 // no changes needed
@@ -429,14 +496,14 @@ impl HaarClassifier {
         })
     }
 
-    fn scan_window(&self, integral_window: nd::ArrayView2<u32>, scale: f64, output_window: &mut nd::ArrayViewMut2<i16>) -> bool{
+    fn scan_window(&self, integral_window: nd::ArrayView2<u32>, scale: f64, output_window: &mut nd::ArrayViewMut2<i16>, x: usize, y: usize, scan_window_size: usize) -> bool{
         let mut failed = false; // let's assume the cascade will work
         for stage in &self.stages{
             let mut stage_sum = 0.;
             let mut i = 0;
             for classifier in &stage.weak_classifiers{
                 // or 'stumps'
-                let feature = classifier.feature.compute_feature(&integral_window, &scale);
+                let feature = classifier.feature.compute_feature(&integral_window, &scale, x, y, scan_window_size);
                 let stddev = 1.; // TODO what should we use here?
                 let threshold = classifier.threshold * stddev;
                 let idx = if feature < threshold {
@@ -445,7 +512,7 @@ impl HaarClassifier {
                 } else {
                     stage_sum += classifier.leaf_values[1];
                     // weak classifier bigger then threshold... draw it!
-                    classifier.feature.draw(output_window, &scale);
+                    classifier.feature.draw(output_window, &scale, x, y, scan_window_size);
                     i+=1;
                     classifier.right
                 };