diff --git a/src/heatmap.rs b/src/heatmap.rs new file mode 100644 index 0000000..522cd5b --- /dev/null +++ b/src/heatmap.rs @@ -0,0 +1,225 @@ +use ndarray as nd; +use image; + +pub enum ColorMaps{ + Binary, + NipySpectral, + TraficLight, +} + +#[derive(Debug)] +pub struct ColorMap{ + pub red: Vec<(f64, f64, f64)>, + pub green: Vec<(f64, f64, f64)>, + pub blue: Vec<(f64, f64, f64)>, +} + +#[derive(Debug)] +pub struct Heatmap{ + pub cm: ColorMap +} + + +impl Heatmap{ + pub fn new(cm: ColorMaps) -> Self{ + Self{ + cm: ColorMap::new(cm) + } + } + + pub fn convert_image(&self, img: image::DynamicImage) -> image::RgbImage { + let gray_img: image::GrayImage = match img { + image::DynamicImage::ImageLuma8(gray_image) => { + gray_image + } + _ => { + img.to_luma() + } + }; + + let mut heatmap_img = image::RgbImage::new(gray_img.width(), gray_img.height()); + let lut_size = 256;// * 256 * 256; + let lut = self.cm.generate_lut(lut_size); + + // info!("LUT: {:?}", lut); + + for pixel in gray_img.enumerate_pixels() { + let l = pixel.2; + let p = image::Rgb(lut[l.0[0] as usize]); + heatmap_img.put_pixel(pixel.0, pixel.1, p); + } + + return heatmap_img; + } +} + +// impl From> for Heatmap { +// fn from(array: nd::Array2) -> Self { + +// } +// } + +// impl From for Heatmap { +// fn from(image: image::DynamicImage) -> Self { +// Self{ +// cm: +// } +// } +// } + +impl ColorMap{ + pub fn new(m: ColorMaps) -> Self { + + let cm = match m { + ColorMaps::Binary => { + Self{ + red: vec![ + (0., 0., 0.), (1., 1., 1.) + ], + green: vec![ + (0., 0., 0.), (1., 1., 1.) + ], + blue: vec![ + (0., 0., 0.), (1., 1., 1.) + ], + } + } + ColorMaps::TraficLight => { + Self{ + red: vec![ + (0., 0., 0.), (0.5, 1., 1.), (1., 1., 1.) + ], + green: vec![ + (0., 0., 0.), (0.5, 1., 1.), (1., 0., 0.) + ], + blue: vec![ + (0., 0., 1.), (0.5, 0., 0.), (1., 0., 0.) + ], + } + } + ColorMaps::NipySpectral => { + Self{ + red: vec![(0.0, 0.0, 0.0), (0.05, 0.4667, 0.4667), + (0.10, 0.5333, 0.5333), (0.15, 0.0, 0.0), + (0.20, 0.0, 0.0), (0.25, 0.0, 0.0), + (0.30, 0.0, 0.0), (0.35, 0.0, 0.0), + (0.40, 0.0, 0.0), (0.45, 0.0, 0.0), + (0.50, 0.0, 0.0), (0.55, 0.0, 0.0), + (0.60, 0.0, 0.0), (0.65, 0.7333, 0.7333), + (0.70, 0.9333, 0.9333), (0.75, 1.0, 1.0), + (0.80, 1.0, 1.0), (0.85, 1.0, 1.0), + (0.90, 0.8667, 0.8667), (0.95, 0.80, 0.80), + (1.0, 0.80, 0.80)], + green: vec![(0.0, 0.0, 0.0), (0.05, 0.0, 0.0), + (0.10, 0.0, 0.0), (0.15, 0.0, 0.0), + (0.20, 0.0, 0.0), (0.25, 0.4667, 0.4667), + (0.30, 0.6000, 0.6000), (0.35, 0.6667, 0.6667), + (0.40, 0.6667, 0.6667), (0.45, 0.6000, 0.6000), + (0.50, 0.7333, 0.7333), (0.55, 0.8667, 0.8667), + (0.60, 1.0, 1.0), (0.65, 1.0, 1.0), + (0.70, 0.9333, 0.9333), (0.75, 0.8000, 0.8000), + (0.80, 0.6000, 0.6000), (0.85, 0.0, 0.0), + (0.90, 0.0, 0.0), (0.95, 0.0, 0.0), + (1.0, 0.80, 0.80)], + blue: vec![(0.0, 0.0, 0.0), (0.05, 0.5333, 0.5333), + (0.10, 0.6000, 0.6000), (0.15, 0.6667, 0.6667), + (0.20, 0.8667, 0.8667), (0.25, 0.8667, 0.8667), + (0.30, 0.8667, 0.8667), (0.35, 0.6667, 0.6667), + (0.40, 0.5333, 0.5333), (0.45, 0.0, 0.0), + (0.5, 0.0, 0.0), (0.55, 0.0, 0.0), + (0.60, 0.0, 0.0), (0.65, 0.0, 0.0), + (0.70, 0.0, 0.0), (0.75, 0.0, 0.0), + (0.80, 0.0, 0.0), (0.85, 0.0, 0.0), + (0.90, 0.0, 0.0), (0.95, 0.0, 0.0), + (1.0, 0.80, 0.80)], + } + } + }; + + return cm; + } + + + /// Similar to MatplotLib LinearSegmentedColormap + /// @see https://github.com/matplotlib/matplotlib/blob/13e3573b721210d84865d148aab7f63cc2fc95a6/lib/matplotlib/colors.py + /// """ + /// Create color map from linear mapping segments + /// segmentdata argument is a dictionary with a red, green and blue + /// entries. Each entry should be a list of *x*, *y0*, *y1* tuples, + /// forming rows in a table. Entries for alpha are optional. + /// Example: suppose you want red to increase from 0 to 1 over + /// the bottom half, green to do the same over the middle half, + /// and blue over the top half. Then you would use:: + /// cdict = {'red': [(0.0, 0.0, 0.0), + /// (0.5, 1.0, 1.0), + /// (1.0, 1.0, 1.0)], + /// 'green': [(0.0, 0.0, 0.0), + /// (0.25, 0.0, 0.0), + /// (0.75, 1.0, 1.0), + /// (1.0, 1.0, 1.0)], + /// 'blue': [(0.0, 0.0, 0.0), + /// (0.5, 0.0, 0.0), + /// (1.0, 1.0, 1.0)]} + /// Each row in the table for a given color is a sequence of + /// *x*, *y0*, *y1* tuples. In each sequence, *x* must increase + /// monotonically from 0 to 1. For any input value *z* falling + /// between *x[i]* and *x[i+1]*, the output value of a given color + /// will be linearly interpolated between *y1[i]* and *y0[i+1]*:: + /// row i: x y0 y1 + /// / + /// / + /// row i+1: x y0 y1 + /// Hence y0 in the first row and y1 in the last row are never used. + /// See Also + /// -------- + /// LinearSegmentedColormap.from_list + /// Static method; factory function for generating a smoothly-varying + /// LinearSegmentedColormap. + /// """ + pub fn generate_lut(&self, N: usize) -> Vec<[u8; 3]> { + let r = Self::interpolate_color(&self.red, N); + let g = Self::interpolate_color(&self.green, N); + let b = Self::interpolate_color(&self.blue, N); + let mut lut = Vec::<[u8;3]>::new(); + + for i in 0..N { + lut.push([r[i], g[i], b[i]]); + } + + lut + } + + pub fn interpolate_color(colors: &Vec<(f64, f64, f64)>, N: usize) -> Vec{ + let step_size = 1./N as f64; + + let mut prev_color: Option<(f64, f64, f64)> = None; + + let mut lut = Vec::::new(); + + for color in colors { + match prev_color { + Some(prev) => { + let steps = (color.0/step_size) as usize - lut.len(); + for i in 0..steps { + let factor = (i + 1) as f64 / steps as f64; + let c = ((prev.2 * (1. - factor) + color.1 * factor) * (N as f64 - 1.)) as u8; + lut.push(c); + } + } + None => { + let steps = (color.0/step_size) as usize; + for i in 0..steps{ + lut.push((color.2 * (N as f64 - 1.)) as u8); + } + } + } + prev_color = Some(color.clone()); + } + // now fill the last bit of the lut with the last color + for pos in lut.len()..N { + lut.push(lut.last().unwrap().clone()); + } + + lut + } +} \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index cc1717f..44c8dfe 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,7 +5,7 @@ use simplelog::*; use nannou::prelude::*; use v4l::{Buffer, CaptureDevice, MappedBufferStream}; - +use image; mod visualhaar; // use std::fs::File; @@ -148,13 +148,11 @@ fn update(_app: &App, _model: &mut Model, _update: Update) { fn view(_app: &App, _model: &Model, frame: Frame){ let draw = _app.draw(); - draw.background().color(PLUM); + draw.background().color(BLACK); let sine = (_app.time / 1.0).sin(); let slowersine = (_app.time / 3.0).sin(); let rotation = _app.time % (2. * PI); let boundary = _app.window_rect(); - let x = map_range(sine, -1.0, 1.0, boundary.left(), boundary.right()); - let y = map_range(slowersine, -1.0, 1.0, boundary.bottom(), boundary.top()); // let texture = wgpu::Texture::load_from_ // let assets = _app.assets_path().unwrap(); @@ -165,12 +163,17 @@ fn view(_app: &App, _model: &Model, frame: Frame){ match &_model.haar_outcome { Some(outcome) => { // let i = outcome.dyn(/); - let texture = wgpu::Texture::from_image(_app, &outcome.dynamic_img); + // let img // ::from(&outcome.dynamic_img); + let img = image::DynamicImage::ImageRgb8(outcome.dynamic_img.to_rgb()).resize(1000, 1000, image::imageops::FilterType::Triangle); + + let texture = wgpu::Texture::from_image(_app, &img); draw.texture(&texture); } _ => {} } - draw.rect().color(STEELBLUE).rotate(rotation).x_y(x,y); + // let x = map_range(sine, -1.0, 1.0, boundary.left(), boundary.right()); + // let y = map_range(slowersine, -1.0, 1.0, boundary.bottom(), boundary.top()); + // draw.rect().color(STEELBLUE).rotate(rotation).x_y(x,y); draw.to_frame(_app, &frame).unwrap(); } diff --git a/src/test.rs b/src/test.rs index ad0c167..2c9a50b 100644 --- a/src/test.rs +++ b/src/test.rs @@ -9,6 +9,7 @@ use v4l::{Buffer, CaptureDevice, MappedBufferStream}; use image; mod visualhaar; +mod heatmap; // use std::fs::File; @@ -30,7 +31,7 @@ fn main() { let sw = Stopwatch::start_new(); - let frame = image::open("/home/ruben/Documents/Projecten/2020/rust/lena_orig.png"); + let frame = image::open("/home/ruben/Documents/Projecten/2020/rust/lena_orig-s.png"); // let vec: Vec = frame.data().to_vec(); @@ -49,6 +50,12 @@ fn main() { // _model.image = Some(nannou::image::DynamicImage::ImageLuma8(ib_bw)); let i = ib.as_rgb8().unwrap().clone(); let image = haar.scan_image(i).unwrap().dynamic_img; + + // let hm = heatmap::Heatmap::new(heatmap::ColorMaps::NipySpectral); + let hm = heatmap::Heatmap::new(heatmap::ColorMaps::TraficLight); + // let hm = heatmap::Heatmap::new(heatmap::ColorMaps::Binary); + let image = hm.convert_image(image); + image.save("/home/ruben/Documents/Projecten/2020/rust/lena_orig-output.png"); info!("Scanning for faces took {}ms", sw.elapsed_ms()); // _model.image = Some(nannou::image::DynamicImage::ImageRgb8(ib)); diff --git a/src/visualhaar.rs b/src/visualhaar.rs index 8aef627..6f4c0c2 100644 --- a/src/visualhaar.rs +++ b/src/visualhaar.rs @@ -4,6 +4,7 @@ use image; use log::{info, trace, warn}; use std::{convert::TryInto, error::Error}; +use stopwatch::{Stopwatch}; use ndarray as nd; /// A haarclasifier based on opencv cascade XML files @@ -60,7 +61,7 @@ impl HaarClassifierFeature{ score } - fn draw(&self, draw_window: &mut nd::ArrayViewMut2, scale: &f64) { + fn draw(&self, draw_window: &mut nd::ArrayViewMut2, scale: &f64) { for rect in &self.rects{ rect.draw(draw_window, scale); } @@ -76,7 +77,7 @@ pub struct HaarClassifierFeatureRect{ width: u8, height: u8, /// weight factor - weight: f64, + weight: i16, } impl HaarClassifierFeatureRect{ @@ -94,12 +95,12 @@ impl HaarClassifierFeatureRect{ let (x1, y1, x2, y2) = self.get_coordinates_for_scale(scale); let sum = (image_window[[y2,x2]] + image_window[[y1,x1]] - image_window[[y1, x2]] - image_window[[y2, x1]]) as f64; - let sum = (sum/(scale*scale)) * self.weight; // normalise: when the window grows, all values of the integral image become bigger by a factor scale-squared + let sum = (sum/(scale*scale)) * self.weight as f64; // normalise: when the window grows, all values of the integral image become bigger by a factor scale-squared return sum; } - fn draw(&self, draw_window: &mut nd::ArrayViewMut2, scale: &f64) { + fn draw(&self, draw_window: &mut nd::ArrayViewMut2, scale: &f64) { let (x1, y1, x2, y2) = self.get_coordinates_for_scale(scale); // TODO how to speed this up? @@ -170,12 +171,13 @@ impl HaarClassifier { // println!("{:?}",rect.text()); let v: Vec<&str> = rect.text().unwrap().split_whitespace().collect(); assert_eq!(v.len(), 5, "Expected values for features: x, y, width, height, weight"); + let w: f64 = v[4].parse()?; rects.push(HaarClassifierFeatureRect{ tl_x: v[0].parse()?, tl_y: v[1].parse()?, width: v[2].parse()?, height: v[3].parse()?, - weight: v[4].parse()?, + weight: w as i16, }); } @@ -334,7 +336,7 @@ impl HaarClassifier { let integral = Self::integral_image(&img_bw); - let mut output_frame: nd::Array2 = nd::Array::zeros(( + let mut output_frame: nd::Array2 = nd::Array::zeros(( img_bw.dimensions().1 as usize, img_bw.dimensions().0 as usize, )); @@ -351,14 +353,16 @@ impl HaarClassifier { let mut count_faces = 0; let mut count_not_faces = 0; while window_size < max_window_size { + + let sw = Stopwatch::start_new(); let scale = (window_size-1) as f64 / self.width as f64; // to calculate a rect, we would need a -1 row, if we ignore that precision and add one at the end: (eg required when an item has width 20 (== feature width)) let scan_window_size = window_size + 1; info!("Window size: {:?} {:?}", window_size, scale); // step by scale.ceil() as this is 1px in the model's size. (small is probably unnecesarily fine-grained) - for x in (0..(img_bw.dimensions().0 as usize - scan_window_size)).step_by(scale.ceil() as usize) { - for y in (0..(img_bw.dimensions().1 as usize - scan_window_size)).step_by(scale.ceil() as usize) { + for x in (0..(img_bw.dimensions().0 as usize - scan_window_size)).step_by((scale * 1.0).ceil() as usize) { + for y in (0..(img_bw.dimensions().1 as usize - scan_window_size)).step_by((scale * 1.0).ceil() as usize) { let window = integral.slice(s![y..y+scan_window_size, x..x+scan_window_size]); let mut output_window = output_frame.slice_mut(s![y..y+scan_window_size, x..x+scan_window_size]); if self.scan_window(window, scale, &mut output_window) { @@ -370,6 +374,8 @@ impl HaarClassifier { } // break; } + + info!("\ttook: {:?}ms", sw.elapsed_ms()); // break; window_size = (window_size as f32 * 1.2) as usize; // TODO make grow-factor variable (now 1.2) @@ -379,14 +385,22 @@ impl HaarClassifier { // test_window += 10.; // Find the largest non-NaN in vector, or NaN otherwise: - let max_output_pixel = output_frame.iter().cloned().fold(0./0., f64::max); - let min_output_pixel = output_frame.iter().cloned().fold(f64::NAN, f64::min); + let max_output_pixel = output_frame.iter().max().unwrap().clone();//when f64: output_frame.iter().cloned().fold(0./0., f64::max); + let min_output_pixel = output_frame.iter().min().unwrap().clone();//when f64: output_frame.iter().cloned().fold(f64::NAN, f64::min); info!("Maximum pixel value in drawing: {:?} / min: {:?}", max_output_pixel, min_output_pixel); info!("Count accepted/rejected windows: {:?}/{:?}", count_faces, count_not_faces); // let max_output_pixel = output_frame.iter().par().unwrap().clone(); output_frame -= min_output_pixel; - output_frame /= (max_output_pixel-min_output_pixel) / 255.; + let pix_diff = (max_output_pixel-min_output_pixel) as f64 / 256.; + if pix_diff.abs() > 1. { + let frac: i16 = if pix_diff.is_sign_positive(){ + pix_diff.ceil() as i16 + } else { + pix_diff.floor() as i16 + }; + output_frame /= frac; + } // let image_frame = output_frame / (max_output_pixel as) // convert to image, thanks to https://stackoverflow.com/a/56762490 @@ -404,10 +418,11 @@ impl HaarClassifier { }) } - fn scan_window(&self, integral_window: nd::ArrayView2, scale: f64, output_window: &mut nd::ArrayViewMut2) -> bool{ + fn scan_window(&self, integral_window: nd::ArrayView2, scale: f64, output_window: &mut nd::ArrayViewMut2) -> bool{ let mut failed = false; // let's assume the cascade will work for stage in &self.stages{ let mut stage_sum = 0.; + let mut i = 0; for classifier in &stage.weak_classifiers{ // or 'stumps' let feature = classifier.feature.compute_feature(&integral_window, &scale); @@ -420,10 +435,14 @@ impl HaarClassifier { stage_sum += classifier.leaf_values[1]; // weak classifier bigger then threshold... draw it! classifier.feature.draw(output_window, &scale); - + i+=1; classifier.right }; + // if i > 2{ + // break; + // } + // classifier.feature.draw(output_window, &scale); } if stage_sum < stage.treshold{