r/emotionhero
1
0
Fork 0
Code Issues 1 Pull requests Releases Wiki Activity

Save image on hit

Browse source
This commit is contained in:
Ruben 2016-09-09 13:00:41 +01:00
parent 5794cbae4c
commit c856ed3ecc
10 changed files with 157 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
app/build.gradle
View file

@ -12,6 +12,9 @@ android {
versionCode 1
versionName '0.1'
testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner"
renderscriptTargetApi 16
renderscriptSupportModeEnabled true
}
buildTypes {
release {
@ -35,5 +38,7 @@ dependencies {
compile 'com.android.support:design:23.4.0'
compile 'com.loopj.android:android-async-http:1.4.9'
compile 'org.ocpsoft.prettytime:prettytime:4.0.1.Final'
compile 'io.github.silvaren:easyrs:0.5.3'
testCompile 'junit:junit:4.12'
}

16
app/src/main/java/com/rubenvandeven/emotionhero/GameOpenHelper.java
View file

@ -20,7 +20,7 @@ import java.util.Date;
* adb pull /sdcard/emotionhero.db
*/
public class GameOpenHelper extends SQLiteOpenHelper {
private static final int DATABASE_VERSION = 2;
private static final int DATABASE_VERSION = 3;
private static final String GAME_TABLE_NAME = "games";
private static final String GAME_TABLE_CREATE =
"CREATE TABLE " + GAME_TABLE_NAME + " (" +
@ -146,9 +146,10 @@ public class GameOpenHelper extends SQLiteOpenHelper {
"point_32y VARCHAR(20)," +
"point_33x VARCHAR(20)," +
"point_33y VARCHAR(20)," +
"image BLOB," +
"PRIMARY KEY (id));" ;
private static final String[] HIT_PROJECTION = {
"id","game_id", "target_index","score","remoteId","glasses","ethnicity","age","gender","anger","contempt","disgust","fear","joy","sadness","surprise","roll","pitch","yaw","inter_ocular_distance","mouth_open","lip_press","brow_raise","nose_wrinkler","lip_depressor","brow_furrow","attention","smile","inner_brow_raiser","chin_raiser","smirk","lip_suck","upper_lip_raiser","lip_pucker","eye_closure","engagement","valence","point_0x","point_0y","point_1x","point_1y","point_2x","point_2y","point_3x","point_3y","point_4x","point_4y","point_5x","point_5y","point_6x","point_6y","point_7x","point_7y","point_8x","point_8y","point_9x","point_9y","point_10x","point_10y","point_11x","point_11y","point_12x","point_12y","point_13x","point_13y","point_14x","point_14y","point_15x","point_15y","point_16x","point_16y","point_17x","point_17y","point_18x","point_18y","point_19x","point_19y","point_20x","point_20y","point_21x","point_21y","point_22x","point_22y","point_23x","point_23y","point_24x","point_24y","point_25x","point_25y","point_26x","point_26y","point_27x","point_27y","point_28x","point_28y","point_29x","point_29y","point_30x","point_30y","point_31x","point_31y","point_32x","point_32y","point_33x","point_33y"
"id","game_id", "target_index","score","remoteId","glasses","ethnicity","age","gender","anger","contempt","disgust","fear","joy","sadness","surprise","roll","pitch","yaw","inter_ocular_distance","mouth_open","lip_press","brow_raise","nose_wrinkler","lip_depressor","brow_furrow","attention","smile","inner_brow_raiser","chin_raiser","smirk","lip_suck","upper_lip_raiser","lip_pucker","eye_closure","engagement","valence","point_0x","point_0y","point_1x","point_1y","point_2x","point_2y","point_3x","point_3y","point_4x","point_4y","point_5x","point_5y","point_6x","point_6y","point_7x","point_7y","point_8x","point_8y","point_9x","point_9y","point_10x","point_10y","point_11x","point_11y","point_12x","point_12y","point_13x","point_13y","point_14x","point_14y","point_15x","point_15y","point_16x","point_16y","point_17x","point_17y","point_18x","point_18y","point_19x","point_19y","point_20x","point_20y","point_21x","point_21y","point_22x","point_22y","point_23x","point_23y","point_24x","point_24y","point_25x","point_25y","point_26x","point_26y","point_27x","point_27y","point_28x","point_28y","point_29x","point_29y","point_30x","point_30y","point_31x","point_31y","point_32x","point_32y","point_33x","point_33y", "image"
};
private Context context;
@ -175,6 +176,9 @@ public class GameOpenHelper extends SQLiteOpenHelper {
db.execSQL(GAME_TABLE_CREATE);
db.execSQL(HIT_TABLE_CREATE);
}
if(oldVersion == 2 && newVersion == 3) {
db.execSQL("ALTER TABLE hits ADD image BLOB;");
}
}
public void insertGame(Game game) {
@ -299,6 +303,11 @@ public class GameOpenHelper extends SQLiteOpenHelper {
values.put("remoteId", hit.remoteId);
}
byte[] frameBytes = hit.getFrameAsByteArray();
if(frameBytes != null) {
values.put("image", frameBytes);
}
// Insert the new row, returning the primary key value of the new row
long newRowId = db.insert(HIT_TABLE_NAME, null, values);
hit.id = newRowId;
@ -344,6 +353,9 @@ public class GameOpenHelper extends SQLiteOpenHelper {
ContentValues values = new ContentValues();
values.put("remoteId", hit.remoteId);
// reset image after sync to avoid to much data used on phone
// values.putNull("image");
// Which row to update, based on the title
String selection = "id = ?";
String[] selectionArgs = { Long.toString(hit.id) };

5
app/src/main/java/com/rubenvandeven/emotionhero/GamingActivity.java
View file

@ -6,6 +6,7 @@ import android.app.Dialog;
import android.content.DialogInterface;
import android.content.Intent;
import android.content.pm.PackageManager;
import android.graphics.Typeface;
import android.media.AudioAttributes;
import android.media.AudioManager;
import android.media.SoundPool;
@ -155,6 +156,9 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL
StoryDialogFragment storyDialog = new StoryDialogFragment();
storyDialog.show(getSupportFragmentManager(), "StoryDialog");
Typeface font = Typeface.createFromAsset(getAssets(), "unifont-9.0.02.ttf");
mContentView.setTypeface(font);
}
public void startGame() {
@ -248,6 +252,7 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL
Face face = list.get(0);
currentScenario.setCurrentFace(face);
scenarioView.setCurrentAttributeScoresForFace(face);
currentScenario.currentFrame = (Frame.ByteArrayFrame) frame;
}
}

61
app/src/main/java/com/rubenvandeven/emotionhero/Hit.java
View file

@ -1,12 +1,17 @@
package com.rubenvandeven.emotionhero;
import android.database.Cursor;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.graphics.PointF;
import android.graphics.RectF;
import android.util.Base64;
import com.affectiva.android.affdex.sdk.detector.Face;
import com.google.gson.annotations.Expose;
import java.util.ArrayList;
import java.io.ByteArrayOutputStream;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
@ -38,10 +43,12 @@ public class Hit {
public String remoteId;
public Bitmap frame;
/**
* Constructor for game moment
*/
public Hit(Scenario.Target target, Game game, Face face, float score) {
public Hit(Scenario.Target target, Game game, Face face, float score, Bitmap frame) {
this.score = score;
this.target = target;
this.game = game;
@ -60,6 +67,10 @@ public class Hit {
this.points = face.getFacePoints().clone();
this.frame = frame;
// this in a way is strange behaviour:
// preferably do the game.addHit after creating hit, and call hit.game = game from there.
game.addHit(this);
}
@ -125,6 +136,52 @@ public class Hit {
this.points[32] = new PointF(c.getFloat(c.getColumnIndex("point_32x")), c.getFloat(c.getColumnIndex("point_32y")));
this.points[33] = new PointF(c.getFloat(c.getColumnIndex("point_33x")), c.getFloat(c.getColumnIndex("point_33y")));
byte[] image = c.getBlob(c.getColumnIndex("image"));
if(image != null)
{
frame = BitmapFactory.decodeByteArray(image, 0, image.length);
}
game.addHit(this);
}
public static RectF getBoundingboxForPoints(PointF[] points) {
// a bit of a hack to be as quick as possible
// we know which point is where on the face, however, the face might be tilted:
float[] minXes = {points[0].x,points[1].x,points[5].x};
float[] maxXes = {points[4].x,points[3].x,points[10].x};
float[] maxYs = {points[1].y,points[2].y,points[3].y};
float[] minYs = {points[5].y,points[6].y,points[7].y,points[8].y,points[9].y,points[10].y};
Arrays.sort(minXes);
Arrays.sort(maxXes);
Arrays.sort(maxYs);
Arrays.sort(minYs);
return new RectF(minXes[0], minYs[0], maxXes[maxXes.length-1], maxYs[maxYs.length-1]);
}
// to store in db
public static String bitmapToBase64(Bitmap bitmap) {
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
// bitmap.compress(Bitmap.CompressFormat.PNG, 100, byteArrayOutputStream);
bitmap.compress(Bitmap.CompressFormat.JPEG, 90, byteArrayOutputStream);
byte[] byteArray = byteArrayOutputStream .toByteArray();
return Base64.encodeToString(byteArray, Base64.DEFAULT);
}
// to retreive from db
public static Bitmap base64ToBitmap(String b64) {
byte[] imageAsBytes = Base64.decode(b64.getBytes(), Base64.DEFAULT);
return BitmapFactory.decodeByteArray(imageAsBytes, 0, imageAsBytes.length);
}
public byte[] getFrameAsByteArray() {
if(frame == null)
return null;
ByteArrayOutputStream stream = new ByteArrayOutputStream();
frame.compress(Bitmap.CompressFormat.JPEG, 90, stream);
return stream.toByteArray();
}
}

3
app/src/main/java/com/rubenvandeven/emotionhero/MenuActivity.java
View file

@ -15,6 +15,9 @@ import android.widget.Button;
import android.widget.ImageView;
import android.widget.Toast;
/**
* @deprecated Now done using MirrorMenuActivity
*/
public class MenuActivity extends AppCompatActivity {
Player player;

22
app/src/main/java/com/rubenvandeven/emotionhero/MirrorMenuActivity.java
View file

@ -6,7 +6,8 @@ import android.animation.ValueAnimator;
import android.app.ProgressDialog;
import android.content.Intent;
import android.content.pm.PackageManager;
import android.graphics.Color;
import android.graphics.Bitmap;
import android.support.v8.renderscript.RenderScript;
import android.graphics.Typeface;
import android.support.v4.app.ActivityCompat;
import android.support.v7.app.AppCompatActivity;
@ -28,10 +29,10 @@ import com.affectiva.android.affdex.sdk.detector.CameraDetector;
import com.affectiva.android.affdex.sdk.detector.Detector;
import com.affectiva.android.affdex.sdk.detector.Face;
import org.w3c.dom.Text;
import java.util.List;
import io.github.silvaren.easyrs.tools.Nv21Image;
public class MirrorMenuActivity extends AppCompatActivity implements Detector.ImageListener, CameraDetector.CameraEventListener, Detector.FaceListener {
final static String LOG_TAG = "EmotionHero-Mirror";
@ -52,6 +53,8 @@ public class MirrorMenuActivity extends AppCompatActivity implements Detector.Im
ProgressDialog loadGameDialog;
RenderScript rs;
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
@ -72,6 +75,7 @@ public class MirrorMenuActivity extends AppCompatActivity implements Detector.Im
startButton .setTypeface(font);
highscoresButton .setTypeface(font);
creditsButton .setTypeface(font);
messageText.setTypeface(font);
//We create a custom SurfaceView that resizes itself to match the aspect ratio of the incoming camera frames
cameraPreview = new SurfaceView(this) {
@ -183,6 +187,8 @@ public class MirrorMenuActivity extends AppCompatActivity implements Detector.Im
startActivity(intent);
}
});
rs = RenderScript.create(this);
}
@Override
@ -227,8 +233,8 @@ public class MirrorMenuActivity extends AppCompatActivity implements Detector.Im
detector.setDetectAllEmotions(true);
detector.setDetectAllAppearances(false);
detector.setDetectAllEmojis(false);
detector.setDetectAllExpressions(true);
detector.setMaxProcessRate(20);
detector.setDetectAllExpressions(false);
detector.setMaxProcessRate(12);
detector.setImageListener(this);
detector.setOnCameraEventListener(this);
@ -252,10 +258,10 @@ public class MirrorMenuActivity extends AppCompatActivity implements Detector.Im
/**
* Detector callback gives the faces found so we can match their hits to the given scenario.
*/
public void onImageResults(List<Face> list, Frame frame, float timestamp) {if (list == null)
return;
if (list.size() == 0) {
public void onImageResults(List<Face> list, Frame frame, float timestamp) {
if (list == null || list.size() == 0) {
setText("Show me your face");
return;
} else {
hideText();
Face face = list.get(0);

56
app/src/main/java/com/rubenvandeven/emotionhero/Scenario.java
View file

@ -1,8 +1,14 @@
package com.rubenvandeven.emotionhero;
import android.graphics.Bitmap;
import android.graphics.Canvas;
import android.graphics.Color;
import android.graphics.Paint;
import android.graphics.RectF;
import android.support.v8.renderscript.RenderScript;
import android.util.Log;
import com.affectiva.android.affdex.sdk.Frame;
import com.affectiva.android.affdex.sdk.detector.Face;
import java.util.ArrayList;
@ -12,6 +18,8 @@ import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import io.github.silvaren.easyrs.tools.Nv21Image;
/**
* Created by ruben on 16/08/16.
*/
@ -20,7 +28,7 @@ public class Scenario {
public int id;
public Bitmap currentFrameBitmap;
public Frame.ByteArrayFrame currentFrame;
public static final int LVL_NONE = 0;
public static final int LVL_ANGER = 1;
@ -42,9 +50,16 @@ public class Scenario {
/**
* Minimum score to be able to pass to the next level.
* (check minimumScore || minimumAchievements)
*/
float minimumScore = 0;
/**
* Number of achievements required to have, before being able to go to next level
* (check minimumScore || minimumAchievements)
*/
float minimumAchievements= 0;
/**
* If a game is beign played.
*/
@ -72,6 +87,10 @@ public class Scenario {
int maxScore = 0;
protected RenderScript rs;
protected Paint squarePaint;
/**
* The scorres in this moment, as to draw them on the screen.
* Indexes are Emotion ordinals
@ -118,6 +137,10 @@ public class Scenario {
};
timer.schedule(tickTask, 0, 1000/DESIRED_FPS);
this.game = new Game(null, this, 0, new Date(), null);
rs = RenderScript.create(_activity);
squarePaint = new Paint();
squarePaint.setColor(Color.YELLOW);
}
/**
@ -144,13 +167,38 @@ public class Scenario {
float scored_value = target.emotion.getValueFromFace(currentFace);
float required_value = target.value;
float score = 100 - Math.abs(scored_value-required_value);
Hit hit = new Hit(target, game, currentFace, score);
//
_activity.sound.play(_activity.soundIds.get(_activity.SOUND_SCORE), 1, 1,
1,0, hit.score / 200f + 0.5f ); // play back the sound slower
1,0, score / 200f + 0.5f ); // play back the sound slower
// depending on hit value
target.isHit = true;
Bitmap outputBitmap = null;
if(currentFrame != null) {
// convert NV21 byteArrayFrame from camera to RGB bitmap.
Frame.ByteArrayFrame byteArrayFrame = (Frame.ByteArrayFrame) currentFrame;
outputBitmap = Nv21Image.nv21ToBitmap(rs, byteArrayFrame.getByteArray(), byteArrayFrame.getWidth(), byteArrayFrame.getHeight());
Canvas canvas = new Canvas(outputBitmap);
RectF rect = Hit.getBoundingboxForPoints(currentFace.getFacePoints());
canvas.drawRect(rect, squarePaint);
/*
Also possible: only grayscale image - only loops over first part of the bytearray
int[] pixels;
int p;
int size = width*height;
for(int i = 0; i < size; i++) {
p = data[i] & 0xFF;
pixels[i] = 0xff000000 | p<<16 | p<<8 | p;
}
Bitmap bm = Bitmap.createBitmap(pixels, width, height, Bitmap.Config.ARGB_8888);
*/
}
Hit hit = new Hit(target, game, currentFace, score, outputBitmap);
}
}

6
app/src/main/res/layout/activity_gaming.xml
View file

@ -27,11 +27,7 @@
android:textColor="#ffffff"
android:textSize="50sp"
android:textStyle="bold"
android:paddingBottom="100dp"
android:shadowColor="@android:color/black"
android:shadowDx="0"
android:shadowDy="0"
android:shadowRadius="20" />
android:paddingBottom="100dp" />
<FrameLayout

3
app/src/main/res/layout/activity_mirror_menu.xml
View file

@ -69,7 +69,7 @@
android:keepScreenOn="true"
android:text="@string/dummy_content"
android:textColor="@color/textPrimary"
android:textSize="36sp"
android:textSize="30sp"
android:textStyle="bold"
android:paddingBottom="30dp"
android:shadowColor="@android:color/black"
@ -77,7 +77,6 @@
android:shadowDy="0"
android:layout_alignParentBottom="true"
android:shadowRadius="20"
android:typeface="normal"
android:fontFamily="sans-serif-condensed" />

5
app/src/main/res/layout/activity_review.xml
View file

@ -143,7 +143,7 @@
android:layout_marginBottom="5dp" />
<TextView
android:layout_width="wrap_content"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_below="@+id/improveTitle"
android:paddingLeft="@dimen/fab_margin"
@ -151,7 +151,8 @@
android:paddingBottom="@dimen/fab_margin"
android:id="@+id/hintText"
android:textColor="@color/textHighlight"
tools:text="You hit a 50% on the 2nd target, to get a 63% score, raise your brows 2% more." />
tools:text="You hit a 50% on the 2nd target, to get a 63% score, raise your brows 2% more."
android:textSize="18sp" />
<RelativeLayout
android:layout_width="match_parent"