Save image on hit

app/build.gradle

@@ -12,6 +12,9 @@ android {
         versionCode 1
         versionName '0.1'
         testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner"
+
+        renderscriptTargetApi 16
+        renderscriptSupportModeEnabled true
     }
     buildTypes {
         release {
@@ -35,5 +38,7 @@ dependencies {
     compile 'com.android.support:design:23.4.0'
     compile 'com.loopj.android:android-async-http:1.4.9'
     compile 'org.ocpsoft.prettytime:prettytime:4.0.1.Final'
+    compile 'io.github.silvaren:easyrs:0.5.3'
     testCompile 'junit:junit:4.12'
+
 }

app/src/main/java/com/rubenvandeven/emotionhero/GameOpenHelper.java

@@ -20,7 +20,7 @@ import java.util.Date;
  * adb pull /sdcard/emotionhero.db
  */
 public class GameOpenHelper extends SQLiteOpenHelper {
-    private static final int DATABASE_VERSION = 2;
+    private static final int DATABASE_VERSION = 3;
     private static final String GAME_TABLE_NAME = "games";
     private static final String GAME_TABLE_CREATE =
             "CREATE TABLE " + GAME_TABLE_NAME + " (" +
@@ -146,9 +146,10 @@ public class GameOpenHelper extends SQLiteOpenHelper {
                     "point_32y VARCHAR(20)," +
                     "point_33x VARCHAR(20)," +
                     "point_33y VARCHAR(20)," +
+                    "image BLOB," +
                     "PRIMARY KEY (id));" ;
     private static final String[] HIT_PROJECTION = {
-            "id","game_id", "target_index","score","remoteId","glasses","ethnicity","age","gender","anger","contempt","disgust","fear","joy","sadness","surprise","roll","pitch","yaw","inter_ocular_distance","mouth_open","lip_press","brow_raise","nose_wrinkler","lip_depressor","brow_furrow","attention","smile","inner_brow_raiser","chin_raiser","smirk","lip_suck","upper_lip_raiser","lip_pucker","eye_closure","engagement","valence","point_0x","point_0y","point_1x","point_1y","point_2x","point_2y","point_3x","point_3y","point_4x","point_4y","point_5x","point_5y","point_6x","point_6y","point_7x","point_7y","point_8x","point_8y","point_9x","point_9y","point_10x","point_10y","point_11x","point_11y","point_12x","point_12y","point_13x","point_13y","point_14x","point_14y","point_15x","point_15y","point_16x","point_16y","point_17x","point_17y","point_18x","point_18y","point_19x","point_19y","point_20x","point_20y","point_21x","point_21y","point_22x","point_22y","point_23x","point_23y","point_24x","point_24y","point_25x","point_25y","point_26x","point_26y","point_27x","point_27y","point_28x","point_28y","point_29x","point_29y","point_30x","point_30y","point_31x","point_31y","point_32x","point_32y","point_33x","point_33y"
+            "id","game_id", "target_index","score","remoteId","glasses","ethnicity","age","gender","anger","contempt","disgust","fear","joy","sadness","surprise","roll","pitch","yaw","inter_ocular_distance","mouth_open","lip_press","brow_raise","nose_wrinkler","lip_depressor","brow_furrow","attention","smile","inner_brow_raiser","chin_raiser","smirk","lip_suck","upper_lip_raiser","lip_pucker","eye_closure","engagement","valence","point_0x","point_0y","point_1x","point_1y","point_2x","point_2y","point_3x","point_3y","point_4x","point_4y","point_5x","point_5y","point_6x","point_6y","point_7x","point_7y","point_8x","point_8y","point_9x","point_9y","point_10x","point_10y","point_11x","point_11y","point_12x","point_12y","point_13x","point_13y","point_14x","point_14y","point_15x","point_15y","point_16x","point_16y","point_17x","point_17y","point_18x","point_18y","point_19x","point_19y","point_20x","point_20y","point_21x","point_21y","point_22x","point_22y","point_23x","point_23y","point_24x","point_24y","point_25x","point_25y","point_26x","point_26y","point_27x","point_27y","point_28x","point_28y","point_29x","point_29y","point_30x","point_30y","point_31x","point_31y","point_32x","point_32y","point_33x","point_33y", "image"
     };
 
     private Context context;
@@ -175,6 +176,9 @@ public class GameOpenHelper extends SQLiteOpenHelper {
             db.execSQL(GAME_TABLE_CREATE);
             db.execSQL(HIT_TABLE_CREATE);
         }
+        if(oldVersion == 2 && newVersion == 3) {
+            db.execSQL("ALTER TABLE hits ADD image BLOB;");
+        }
     }
 
     public void insertGame(Game game) {
@@ -299,6 +303,11 @@ public class GameOpenHelper extends SQLiteOpenHelper {
             values.put("remoteId", hit.remoteId);
         }
 
+        byte[] frameBytes = hit.getFrameAsByteArray();
+        if(frameBytes != null) {
+            values.put("image", frameBytes);
+        }
+
         // Insert the new row, returning the primary key value of the new row
         long newRowId = db.insert(HIT_TABLE_NAME, null, values);
         hit.id = newRowId;
@@ -344,6 +353,9 @@ public class GameOpenHelper extends SQLiteOpenHelper {
         ContentValues values = new ContentValues();
         values.put("remoteId", hit.remoteId);
 
+        // reset image after sync to avoid to much data used on phone
+//        values.putNull("image");
+
         // Which row to update, based on the title
         String selection = "id = ?";
         String[] selectionArgs = { Long.toString(hit.id) };

app/src/main/java/com/rubenvandeven/emotionhero/GamingActivity.java

@@ -6,6 +6,7 @@ import android.app.Dialog;
 import android.content.DialogInterface;
 import android.content.Intent;
 import android.content.pm.PackageManager;
+import android.graphics.Typeface;
 import android.media.AudioAttributes;
 import android.media.AudioManager;
 import android.media.SoundPool;
@@ -155,6 +156,9 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL
         StoryDialogFragment storyDialog = new StoryDialogFragment();
         storyDialog.show(getSupportFragmentManager(), "StoryDialog");
 
+        Typeface font = Typeface.createFromAsset(getAssets(), "unifont-9.0.02.ttf");
+        mContentView.setTypeface(font);
+
     }
 
     public void startGame() {
@@ -248,6 +252,7 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL
             Face face = list.get(0);
             currentScenario.setCurrentFace(face);
             scenarioView.setCurrentAttributeScoresForFace(face);
+            currentScenario.currentFrame = (Frame.ByteArrayFrame) frame;
         }
     }
 

app/src/main/java/com/rubenvandeven/emotionhero/Hit.java

@@ -1,12 +1,17 @@
 package com.rubenvandeven.emotionhero;
 
 import android.database.Cursor;
+import android.graphics.Bitmap;
+import android.graphics.BitmapFactory;
 import android.graphics.PointF;
+import android.graphics.RectF;
+import android.util.Base64;
 
 import com.affectiva.android.affdex.sdk.detector.Face;
 import com.google.gson.annotations.Expose;
 
-import java.util.ArrayList;
+import java.io.ByteArrayOutputStream;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -38,10 +43,12 @@ public class Hit {
 
     public String remoteId;
 
+    public Bitmap frame;
+
     /**
      * Constructor for game moment
      */
-    public Hit(Scenario.Target target, Game game, Face face, float score) {
+    public Hit(Scenario.Target target, Game game, Face face, float score, Bitmap frame) {
         this.score = score;
         this.target = target;
         this.game = game;
@@ -60,6 +67,10 @@ public class Hit {
 
         this.points = face.getFacePoints().clone();
 
+        this.frame = frame;
+
+        // this in a way is strange behaviour:
+        // preferably do the game.addHit after creating hit, and call hit.game = game from there.
         game.addHit(this);
     }
 
@@ -125,6 +136,52 @@ public class Hit {
         this.points[32] = new PointF(c.getFloat(c.getColumnIndex("point_32x")), c.getFloat(c.getColumnIndex("point_32y")));
         this.points[33] = new PointF(c.getFloat(c.getColumnIndex("point_33x")), c.getFloat(c.getColumnIndex("point_33y")));
 
+        byte[] image = c.getBlob(c.getColumnIndex("image"));
+        if(image != null)
+        {
+            frame = BitmapFactory.decodeByteArray(image, 0, image.length);
+        }
+
         game.addHit(this);
     }
+
+    public static RectF getBoundingboxForPoints(PointF[] points) {
+        // a bit of a hack to be as quick as possible
+        // we know which point is where on the face, however, the face might be tilted:
+        float[] minXes = {points[0].x,points[1].x,points[5].x};
+        float[] maxXes = {points[4].x,points[3].x,points[10].x};
+        float[] maxYs = {points[1].y,points[2].y,points[3].y};
+        float[] minYs = {points[5].y,points[6].y,points[7].y,points[8].y,points[9].y,points[10].y};
+
+        Arrays.sort(minXes);
+        Arrays.sort(maxXes);
+        Arrays.sort(maxYs);
+        Arrays.sort(minYs);
+
+        return new RectF(minXes[0], minYs[0], maxXes[maxXes.length-1], maxYs[maxYs.length-1]);
+    }
+
+    // to store in db
+    public static String bitmapToBase64(Bitmap bitmap) {
+        ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+//        bitmap.compress(Bitmap.CompressFormat.PNG, 100, byteArrayOutputStream);
+        bitmap.compress(Bitmap.CompressFormat.JPEG, 90, byteArrayOutputStream);
+        byte[] byteArray = byteArrayOutputStream .toByteArray();
+        return Base64.encodeToString(byteArray, Base64.DEFAULT);
+    }
+
+    // to retreive from db
+    public static Bitmap base64ToBitmap(String b64) {
+        byte[] imageAsBytes = Base64.decode(b64.getBytes(), Base64.DEFAULT);
+        return BitmapFactory.decodeByteArray(imageAsBytes, 0, imageAsBytes.length);
+    }
+
+    public byte[] getFrameAsByteArray() {
+        if(frame == null)
+            return null;
+
+        ByteArrayOutputStream stream = new ByteArrayOutputStream();
+        frame.compress(Bitmap.CompressFormat.JPEG, 90, stream);
+        return stream.toByteArray();
+    }
 }

app/src/main/java/com/rubenvandeven/emotionhero/MenuActivity.java

@@ -15,6 +15,9 @@ import android.widget.Button;
 import android.widget.ImageView;
 import android.widget.Toast;
 
+/**
+ * @deprecated Now done using MirrorMenuActivity
+ */
 public class MenuActivity extends AppCompatActivity {
 
     Player player;

app/src/main/java/com/rubenvandeven/emotionhero/MirrorMenuActivity.java

@@ -6,7 +6,8 @@ import android.animation.ValueAnimator;
 import android.app.ProgressDialog;
 import android.content.Intent;
 import android.content.pm.PackageManager;
-import android.graphics.Color;
+import android.graphics.Bitmap;
+import android.support.v8.renderscript.RenderScript;
 import android.graphics.Typeface;
 import android.support.v4.app.ActivityCompat;
 import android.support.v7.app.AppCompatActivity;
@@ -28,10 +29,10 @@ import com.affectiva.android.affdex.sdk.detector.CameraDetector;
 import com.affectiva.android.affdex.sdk.detector.Detector;
 import com.affectiva.android.affdex.sdk.detector.Face;
 
-import org.w3c.dom.Text;
-
 import java.util.List;
 
+import io.github.silvaren.easyrs.tools.Nv21Image;
+
 public class MirrorMenuActivity extends AppCompatActivity implements Detector.ImageListener,  CameraDetector.CameraEventListener, Detector.FaceListener {
 
     final static String LOG_TAG = "EmotionHero-Mirror";
@@ -52,6 +53,8 @@ public class MirrorMenuActivity extends AppCompatActivity implements Detector.Im
 
     ProgressDialog loadGameDialog;
 
+    RenderScript rs;
+
     @Override
     protected void onCreate(Bundle savedInstanceState) {
         super.onCreate(savedInstanceState);
@@ -72,6 +75,7 @@ public class MirrorMenuActivity extends AppCompatActivity implements Detector.Im
         startButton .setTypeface(font);
         highscoresButton .setTypeface(font);
         creditsButton .setTypeface(font);
+        messageText.setTypeface(font);
 
         //We create a custom SurfaceView that resizes itself to match the aspect ratio of the incoming camera frames
         cameraPreview = new SurfaceView(this) {
@@ -183,6 +187,8 @@ public class MirrorMenuActivity extends AppCompatActivity implements Detector.Im
                 startActivity(intent);
             }
         });
+
+        rs = RenderScript.create(this);
     }
 
     @Override
@@ -227,8 +233,8 @@ public class MirrorMenuActivity extends AppCompatActivity implements Detector.Im
                     detector.setDetectAllEmotions(true);
                     detector.setDetectAllAppearances(false);
                     detector.setDetectAllEmojis(false);
-                    detector.setDetectAllExpressions(true);
+                    detector.setDetectAllExpressions(false);
-                    detector.setMaxProcessRate(20);
+                    detector.setMaxProcessRate(12);
 
                     detector.setImageListener(this);
                     detector.setOnCameraEventListener(this);
@@ -252,10 +258,10 @@ public class MirrorMenuActivity extends AppCompatActivity implements Detector.Im
     /**
      * Detector callback gives the faces found so we can match their hits to the given scenario.
      */
-    public void onImageResults(List<Face> list, Frame frame, float timestamp) {if (list == null)
+    public void onImageResults(List<Face> list, Frame frame, float timestamp) {
-            return;
+        if (list == null || list.size() == 0) {
-        if (list.size() == 0) {
             setText("Show me your face");
+            return;
         } else {
             hideText();
             Face face = list.get(0);

app/src/main/java/com/rubenvandeven/emotionhero/Scenario.java

@@ -1,8 +1,14 @@
 package com.rubenvandeven.emotionhero;
 
 import android.graphics.Bitmap;
+import android.graphics.Canvas;
+import android.graphics.Color;
+import android.graphics.Paint;
+import android.graphics.RectF;
+import android.support.v8.renderscript.RenderScript;
 import android.util.Log;
 
+import com.affectiva.android.affdex.sdk.Frame;
 import com.affectiva.android.affdex.sdk.detector.Face;
 
 import java.util.ArrayList;
@@ -12,6 +18,8 @@ import java.util.Map;
 import java.util.Timer;
 import java.util.TimerTask;
 
+import io.github.silvaren.easyrs.tools.Nv21Image;
+
 /**
  * Created by ruben on 16/08/16.
  */
@@ -20,7 +28,7 @@ public class Scenario {
 
     public int id;
 
-    public Bitmap currentFrameBitmap;
+    public Frame.ByteArrayFrame currentFrame;
 
     public static final int LVL_NONE = 0;
     public static final int LVL_ANGER = 1;
@@ -42,9 +50,16 @@ public class Scenario {
 
     /**
      * Minimum score to be able to pass to the next level.
+     * (check minimumScore || minimumAchievements)
      */
     float minimumScore = 0;
 
+    /**
+     * Number of achievements required to have, before being able to go to next level
+     * (check minimumScore || minimumAchievements)
+     */
+    float minimumAchievements= 0;
+
     /**
      * If a game is beign played.
      */
@@ -72,6 +87,10 @@ public class Scenario {
 
     int maxScore = 0;
 
+    protected RenderScript rs;
+
+    protected Paint squarePaint;
+
     /**
      * The scorres in this moment, as to draw them on the screen.
      * Indexes are Emotion ordinals
@@ -118,6 +137,10 @@ public class Scenario {
         };
         timer.schedule(tickTask, 0, 1000/DESIRED_FPS);
         this.game = new Game(null, this, 0, new Date(), null);
+
+        rs = RenderScript.create(_activity);
+        squarePaint = new Paint();
+        squarePaint.setColor(Color.YELLOW);
     }
 
     /**
@@ -144,13 +167,38 @@ public class Scenario {
                 float scored_value = target.emotion.getValueFromFace(currentFace);
                 float required_value = target.value;
                 float score = 100 - Math.abs(scored_value-required_value);
-                Hit hit = new Hit(target, game, currentFace, score);
 
-                //
                 _activity.sound.play(_activity.soundIds.get(_activity.SOUND_SCORE), 1, 1,
-                                        1,0, hit.score / 200f + 0.5f ); // play back the sound slower
+                                        1,0, score / 200f + 0.5f ); // play back the sound slower
                                                                         // depending on hit value
                 target.isHit = true;
+
+
+
+                Bitmap outputBitmap = null;
+                if(currentFrame != null) {
+                    // convert NV21 byteArrayFrame from camera to RGB bitmap.
+                    Frame.ByteArrayFrame byteArrayFrame = (Frame.ByteArrayFrame) currentFrame;
+                    outputBitmap = Nv21Image.nv21ToBitmap(rs, byteArrayFrame.getByteArray(), byteArrayFrame.getWidth(), byteArrayFrame.getHeight());
+
+                    Canvas canvas = new Canvas(outputBitmap);
+                    RectF rect = Hit.getBoundingboxForPoints(currentFace.getFacePoints());
+                    canvas.drawRect(rect, squarePaint);
+
+                    /*
+                    Also possible: only grayscale image - only loops over first part of the bytearray
+                    int[] pixels;
+                     int p;
+                    int size = width*height;
+                    for(int i = 0; i < size; i++) {
+                        p = data[i] & 0xFF;
+                        pixels[i] = 0xff000000 | p<<16 | p<<8 | p;
+                    }
+                    Bitmap bm = Bitmap.createBitmap(pixels, width, height, Bitmap.Config.ARGB_8888);
+                     */
+                }
+
+                Hit hit = new Hit(target, game, currentFace, score, outputBitmap);
             }
         }
 

app/src/main/res/layout/activity_gaming.xml

@@ -27,11 +27,7 @@
         android:textColor="#ffffff"
         android:textSize="50sp"
         android:textStyle="bold"
-        android:paddingBottom="100dp"
+        android:paddingBottom="100dp" />
-        android:shadowColor="@android:color/black"
-        android:shadowDx="0"
-        android:shadowDy="0"
-        android:shadowRadius="20" />
 
 
     <FrameLayout

app/src/main/res/layout/activity_mirror_menu.xml

@@ -69,7 +69,7 @@
         android:keepScreenOn="true"
         android:text="@string/dummy_content"
         android:textColor="@color/textPrimary"
-        android:textSize="36sp"
+        android:textSize="30sp"
         android:textStyle="bold"
         android:paddingBottom="30dp"
         android:shadowColor="@android:color/black"
@@ -77,7 +77,6 @@
         android:shadowDy="0"
 
         android:layout_alignParentBottom="true"
-        android:shadowRadius="20"
         android:typeface="normal"
         android:fontFamily="sans-serif-condensed" />
 

app/src/main/res/layout/activity_review.xml

@@ -143,7 +143,7 @@
                 android:layout_marginBottom="5dp" />
 
             <TextView
-                android:layout_width="wrap_content"
+                android:layout_width="match_parent"
                 android:layout_height="wrap_content"
                 android:layout_below="@+id/improveTitle"
                 android:paddingLeft="@dimen/fab_margin"
@@ -151,7 +151,8 @@
                 android:paddingBottom="@dimen/fab_margin"
                 android:id="@+id/hintText"
                 android:textColor="@color/textHighlight"
-                tools:text="You hit a 50% on the 2nd target, to get a 63% score, raise your brows 2% more." />
+                tools:text="You hit a 50% on the 2nd target, to get a 63% score, raise your brows 2% more."
+                android:textSize="18sp" />
 
             <RelativeLayout
                 android:layout_width="match_parent"