r/visualhaar
1
0
Fork 0
Code Issues Pull requests Releases 1 Wiki Activity

Compare commits

base: r:ac1aea1d68f346be194e64a3275629a177327f2e
Branches Tags
r:master
r:0.1
..
compare: r:7c7ae29bf9e1390ea304e3708e8f92f6d57f87ff
Branches Tags
r:master
r:0.1

No commits in common. "ac1aea1d68f346be194e64a3275629a177327f2e" and "7c7ae29bf9e1390ea304e3708e8f92f6d57f87ff" have entirely different histories.
ac1aea1d68 ... 7c7ae29bf9

4 changed files with 33 additions and 116 deletions
Show stats Expand all files Collapse all files

15
README.md
View file

@ -1,15 +0,0 @@
# Viola-Jones' Haarcascade visualisation
This is an unusual visualisation of haarcascades. Often, only haarcascade features are drawn. This visualisation however, adds up all matching features in an image. Resulting in a sort of lense which shows the contrasts in the image that the algorithm picks up on most.
* The visual_haarcascades binary ([main.rs](src/main.rs)) uses V4L to capture webcam input and renders to a canvas.
* [test.rs](src/test.rs) binary is an ugly program that looks hard-coded for `haarcascade_frontalface_alt2.xml` and `test.png`. It renders the output to `test-output.png`.
* The library is made for use with eg. Python through cffi. See eg. [this repo](https://git.rubenvandeven.com/r/face_recognition)
## build
Most importantly
```bash
cargo build --lib --release
```

3
src/lib.rs
View file

@ -43,7 +43,6 @@ pub extern "C" fn scan_image(haar: *mut visualhaar::HaarClassifier,
input: *const u8,
buffer: *mut u8,
length: usize,
min_face_factor: usize,
debug: bool) {
if haar.is_null() || input.is_null() || buffer.is_null() {
return;
@ -73,7 +72,7 @@ pub extern "C" fn scan_image(haar: *mut visualhaar::HaarClassifier,
}
}
let image = haar.scan_image(input_frame, &hm,min_face_factor as u32).unwrap().dynamic_img;
let image = haar.scan_image(input_frame, &hm).unwrap().dynamic_img;
let rgb_img = image.to_rgb();
// image.save("/home/ruben/Documents/Projecten/2020/rust/lena_orig-output-lib.png");
info!("Scanning for faces took done");

8
src/test.rs
View file

@ -22,10 +22,11 @@ fn main() {
warn!("test");
let haar = visualhaar::HaarClassifier::from_xml("../haarcascade_frontalface_alt2.xml").unwrap();
let haar = visualhaar::HaarClassifier::from_xml("haarcascade_frontalface_alt2.xml").unwrap();
// println!("Haar: {:?}", haar);
let mut sw = Stopwatch::start_new();
let frame = image::open("test.png");
@ -46,13 +47,12 @@ fn main() {
// _model.image = Some(nannou::image::DynamicImage::ImageLuma8(ib_bw));
let i = ib.as_rgb8().unwrap().clone();
let hm = Some(heatmap::Heatmap::new(heatmap::ColorMaps::Plasma));
let mut sw = Stopwatch::start_new();
let image = haar.scan_image(i, &hm, 4).unwrap().dynamic_img;
info!("Scanning for faces took {}ms", sw.elapsed_ms());
let image = haar.scan_image(i, &hm).unwrap().dynamic_img;
image.save("test-output.png");
// let hm = heatmap::Heatmap::new(heatmap::ColorMaps::NipySpectral);
// let hm = heatmap::Heatmap::new(heatmap::ColorMaps::TraficLight);
info!("Scanning for faces took {}ms", sw.elapsed_ms());
// sw.restart();
// let hm = h;
// let image_hm = hm.convert_image(image);

129
src/visualhaar.rs
View file

@ -52,19 +52,19 @@ pub struct HaarClassifierFeature {
impl HaarClassifierFeature{
fn compute_feature(&self, image_window: &nd::ArrayView2<u32>, scale: &f64, x: usize, y: usize, scan_window_size: usize) -> f64 {
fn compute_feature(&self, image_window: &nd::ArrayView2<u32>, scale: &f64) -> f64 {
let mut score = 0.;
for rect in &self.rects{
score += rect.compute_rect(image_window, x, y);
score += rect.compute_rect(image_window, scale);
}
score
}
fn draw(&self, draw_window: &mut nd::ArrayViewMut2<i16>, scale: &f64, x: usize, y: usize, scan_window_size: usize) {
fn draw(&self, draw_window: &mut nd::ArrayViewMut2<i16>, scale: &f64) {
for rect in &self.rects{
rect.draw(draw_window, scale, x, y, scan_window_size);
rect.draw(draw_window, scale);
}
}
}
@ -91,35 +91,24 @@ impl HaarClassifierFeatureRect{
(x1, y1, x2, y2)
}
fn get_coordinates(&self) -> (usize, usize, usize, usize) {
let x1 = self.tl_x as usize;
let y1 = self.tl_y as usize;
let x2 = x1 + self.width as usize;
let y2 = y1 + self.height as usize;
(x1, y1, x2, y2)
}
/// The feature sum is finally calculated by first summing all values of the pixels inside the rectangle and then multiplying it with the weight factor. Finally, those weighted sums are combined together to yield as a final feature value. Keep in mind that all the coordinates retrieved for a single feature are in relation to the window/model size and not the complete image which is processed.
fn compute_rect(&self, image_window: &nd::ArrayView2<u32>, x: usize, y: usize) -> f64 {
let (x1, y1, x2, y2) = self.get_coordinates();
fn compute_rect(&self, image_window: &nd::ArrayView2<u32>, scale: &f64) -> f64 {
let (x1, y1, x2, y2) = self.get_coordinates_for_scale(scale);
let sum = (image_window[[y+y2,x+x2]] + image_window[[y+y1,x+x1]] - image_window[[y+y1, x+x2]] - image_window[[y+y2, x+x1]]) as f64;
let sum = (sum) * self.weight as f64; // normalise: when the window grows, all values of the integral image become bigger by a factor scale-squared
let sum = (image_window[[y2,x2]] + image_window[[y1,x1]] - image_window[[y1, x2]] - image_window[[y2, x1]]) as f64;
let sum = (sum/(scale*scale)) * self.weight as f64; // normalise: when the window grows, all values of the integral image become bigger by a factor scale-squared
return sum;
}
fn draw(&self, draw_window: &mut nd::ArrayViewMut2<i16>, scale: &f64, x: usize, y: usize, scan_window_size: usize) {
fn draw(&self, draw_window: &mut nd::ArrayViewMut2<i16>, scale: &f64) {
let (x1, y1, x2, y2) = self.get_coordinates_for_scale(scale);
// TODO how to speed this up?
// info!("Draw {} {} {} {} ({:?}),", x1, y1, x2, y2,self);
let mut rect = draw_window.slice_mut(s![y+y1..y+y2, x+x1..x+x2]); // semi slow (initially 500ms)
let mut rect = draw_window.slice_mut(s![y1..y2, x1..x2]); // semi slow (initially 500ms)
rect += self.weight; // super slow (initially 10.000 ms)
// info!("add")
// for x in x1..x2{
// for y in y1..y2{
@ -342,105 +331,57 @@ impl HaarClassifier {
// }
/// take an ImageBuffer and scan it for faces.
/// min_face_factor parameter gives starting size of scan window (frame height / factor). So higher number scans for smaller faces.
pub fn scan_image(&self, frame: image::ImageBuffer<image::Rgb<u8>, Vec<u8>>, heatmap: &Option<heatmap::Heatmap>, min_face_factor: u32) -> Result<Outcome, String> {
let sw = Stopwatch::start_new();
pub fn scan_image(&self, frame: image::ImageBuffer<image::Rgb<u8>, Vec<u8>>, heatmap: &Option<heatmap::Heatmap>) -> Result<Outcome, String> {
let img_bw = image::imageops::grayscale(&frame);
// let mut output_image = image::GrayImage::new(frame.width(), frame.height());
// let integral = Self::integral_image(&img_bw);
let integral = Self::integral_image(&img_bw);
let mut output_frame: nd::Array2<i16> = nd::Array::zeros((
img_bw.height() as usize,
img_bw.width() as usize,
img_bw.dimensions().1 as usize,
img_bw.dimensions().0 as usize,
));
// let mut integral_view = integral.view();
// let mut output_draw_frame = output_frame.view_mut();
// info!("Frame: {:?} {:?}", integral[[0,0]], integral[[integral.dim().0-1,integral.dim().1-1]]);
// let rect = integral.slice(s![3..5, 2..4]);
// let min_size = self.width;
let min_size = frame.height() / min_face_factor; // TODO: Make min face size (or factor) variable
let min_size = frame.height() / 3; // TODO: Make min face size (or factor) variable
let max_window_size = std::cmp::min(img_bw.dimensions().0, img_bw.dimensions().1) as usize;
let mut window_size: usize = min_size.clone() as usize;
let mut count_faces = 0;
let mut count_not_faces = 0;
info!("preprocessing: {:?}ms", sw.elapsed_ms());
let mut loop_time: i64 = 0;
while window_size < max_window_size {
let sw = Stopwatch::start_new();
let scale = (window_size-1) as f64 / self.width as f64;
let img_bw_scaled = image::imageops::resize(
&img_bw,
(img_bw.width() as f64 / scale + 1.) as u32,
(img_bw.height() as f64 / scale + 1.) as u32,
image::imageops::FilterType::CatmullRom
);
let integral = Self::integral_image(&img_bw_scaled);
let mut scaled_output_frame: nd::Array2<i16> = nd::Array::zeros((
img_bw_scaled.dimensions().1 as usize,
img_bw_scaled.dimensions().0 as usize,
));
let integral_view = integral.view();
let mut scaled_output_draw_frame = scaled_output_frame.view_mut();
// to calculate a rect, we would need a -1 row, if we ignore that precision and add one at the end: (eg required when an item has width 20 (== feature width))
let scan_window_size = window_size + 1;
info!("Window size: {:?} {:?} {:?}", window_size, scale, scaled_output_draw_frame.dim());
info!("Window size: {:?} {:?}", window_size, scale);
// step by scale.ceil() as this is 1px in the model's size. (small is probably unnecesarily fine-grained)
for x in (0..(img_bw_scaled.width() as usize - self.width as usize)) {
for y in (0..(img_bw_scaled.height() as usize - self.height as usize)) {
// let window = integral.slice(s![y..y+scan_window_size, x..x+scan_window_size]);
// let mut output_window = output_frame.slice_mut(s![y..y+scan_window_size, x..x+scan_window_size]);
if self.scan_window(integral_view, 1., &mut scaled_output_draw_frame, x, y, window_size) {
for x in (0..(img_bw.dimensions().0 as usize - scan_window_size)).step_by((scale * 1.0).ceil() as usize) {
for y in (0..(img_bw.dimensions().1 as usize - scan_window_size)).step_by((scale * 1.0).ceil() as usize) {
let window = integral.slice(s![y..y+scan_window_size, x..x+scan_window_size]);
let mut output_window = output_frame.slice_mut(s![y..y+scan_window_size, x..x+scan_window_size]);
if self.scan_window(window, scale, &mut output_window) {
count_faces += 1;
} else {
count_not_faces += 1;
}
}
}
for x in 0..img_bw.width() {
for y in 0..img_bw.height() {
let src_x = (x as f64 / scale) as usize;
let src_y = (y as f64 / scale) as usize;
let weight = scaled_output_draw_frame[[src_y, src_x]];
// let weight = img_bw_scaled.get_pixel(src_x, src_y).0.first().unwrap().clone() as i16;
// info!("Pixel: {:?} {:?} {:?} {:?} {:?} {:?} {:?}", x, y, output_frame.dim(), scale, src_x, src_y, img_bw_scaled.dimensions());
output_frame[[y as usize, x as usize]] += weight;
}
}
let elapsed = sw.elapsed_ms();
info!("\ttook: {:?}ms", elapsed);
// break;
loop_time += elapsed;
}
// break;
}
info!("\ttook: {:?}ms", sw.elapsed_ms());
// break;
window_size = (window_size as f32 * 1.2) as usize; // TODO make grow-factor variable (now 1.2)
}
info!("Looping took: {:?}", loop_time);
let sw = Stopwatch::start_new();
// let mut test_window = output_frame.slice_mut(s![10..20,40..50]);
// test_window += 10.;
@ -472,18 +413,10 @@ impl HaarClassifier {
// let dynamic = image::DynamicImage::ImageLuma8(img_bw);
let dynamic = image::DynamicImage::ImageLuma8(final_img);
info!("postprocessing: {:?}ms", sw.elapsed_ms());
let dynamic = match heatmap {
Some(hm) => {
// TODO remove intermediate DynamicImage conversin
let sw = Stopwatch::start_new();
let i = image::DynamicImage::ImageRgb8(hm.convert_image(dynamic));
info!("heatmap: {:?}ms", sw.elapsed_ms());
i
image::DynamicImage::ImageRgb8(hm.convert_image(dynamic))
}
None => {
// no changes needed
@ -496,14 +429,14 @@ impl HaarClassifier {
})
}
fn scan_window(&self, integral_window: nd::ArrayView2<u32>, scale: f64, output_window: &mut nd::ArrayViewMut2<i16>, x: usize, y: usize, scan_window_size: usize) -> bool{
fn scan_window(&self, integral_window: nd::ArrayView2<u32>, scale: f64, output_window: &mut nd::ArrayViewMut2<i16>) -> bool{
let mut failed = false; // let's assume the cascade will work
for stage in &self.stages{
let mut stage_sum = 0.;
let mut i = 0;
for classifier in &stage.weak_classifiers{
// or 'stumps'
let feature = classifier.feature.compute_feature(&integral_window, &scale, x, y, scan_window_size);
let feature = classifier.feature.compute_feature(&integral_window, &scale);
let stddev = 1.; // TODO what should we use here?
let threshold = classifier.threshold * stddev;
let idx = if feature < threshold {
@ -512,7 +445,7 @@ impl HaarClassifier {
} else {
stage_sum += classifier.leaf_values[1];
// weak classifier bigger then threshold... draw it!
classifier.feature.draw(output_window, &scale, x, y, scan_window_size);
classifier.feature.draw(output_window, &scale);
i+=1;
classifier.right
};