diff --git a/app/build.gradle b/app/build.gradle index 2b2ba35..6c5e575 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -12,6 +12,9 @@ android { versionCode 1 versionName '0.1' testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner" + + renderscriptTargetApi 16 + renderscriptSupportModeEnabled true } buildTypes { release { @@ -35,5 +38,7 @@ dependencies { compile 'com.android.support:design:23.4.0' compile 'com.loopj.android:android-async-http:1.4.9' compile 'org.ocpsoft.prettytime:prettytime:4.0.1.Final' + compile 'io.github.silvaren:easyrs:0.5.3' testCompile 'junit:junit:4.12' + } diff --git a/app/src/main/java/com/rubenvandeven/emotionhero/GameOpenHelper.java b/app/src/main/java/com/rubenvandeven/emotionhero/GameOpenHelper.java index fcef55e..836557b 100644 --- a/app/src/main/java/com/rubenvandeven/emotionhero/GameOpenHelper.java +++ b/app/src/main/java/com/rubenvandeven/emotionhero/GameOpenHelper.java @@ -20,7 +20,7 @@ import java.util.Date; * adb pull /sdcard/emotionhero.db */ public class GameOpenHelper extends SQLiteOpenHelper { - private static final int DATABASE_VERSION = 2; + private static final int DATABASE_VERSION = 3; private static final String GAME_TABLE_NAME = "games"; private static final String GAME_TABLE_CREATE = "CREATE TABLE " + GAME_TABLE_NAME + " (" + @@ -146,9 +146,10 @@ public class GameOpenHelper extends SQLiteOpenHelper { "point_32y VARCHAR(20)," + "point_33x VARCHAR(20)," + "point_33y VARCHAR(20)," + + "image BLOB," + "PRIMARY KEY (id));" ; private static final String[] HIT_PROJECTION = { - "id","game_id", "target_index","score","remoteId","glasses","ethnicity","age","gender","anger","contempt","disgust","fear","joy","sadness","surprise","roll","pitch","yaw","inter_ocular_distance","mouth_open","lip_press","brow_raise","nose_wrinkler","lip_depressor","brow_furrow","attention","smile","inner_brow_raiser","chin_raiser","smirk","lip_suck","upper_lip_raiser","lip_pucker","eye_closure","engagement","valence","point_0x","point_0y","point_1x","point_1y","point_2x","point_2y","point_3x","point_3y","point_4x","point_4y","point_5x","point_5y","point_6x","point_6y","point_7x","point_7y","point_8x","point_8y","point_9x","point_9y","point_10x","point_10y","point_11x","point_11y","point_12x","point_12y","point_13x","point_13y","point_14x","point_14y","point_15x","point_15y","point_16x","point_16y","point_17x","point_17y","point_18x","point_18y","point_19x","point_19y","point_20x","point_20y","point_21x","point_21y","point_22x","point_22y","point_23x","point_23y","point_24x","point_24y","point_25x","point_25y","point_26x","point_26y","point_27x","point_27y","point_28x","point_28y","point_29x","point_29y","point_30x","point_30y","point_31x","point_31y","point_32x","point_32y","point_33x","point_33y" + "id","game_id", "target_index","score","remoteId","glasses","ethnicity","age","gender","anger","contempt","disgust","fear","joy","sadness","surprise","roll","pitch","yaw","inter_ocular_distance","mouth_open","lip_press","brow_raise","nose_wrinkler","lip_depressor","brow_furrow","attention","smile","inner_brow_raiser","chin_raiser","smirk","lip_suck","upper_lip_raiser","lip_pucker","eye_closure","engagement","valence","point_0x","point_0y","point_1x","point_1y","point_2x","point_2y","point_3x","point_3y","point_4x","point_4y","point_5x","point_5y","point_6x","point_6y","point_7x","point_7y","point_8x","point_8y","point_9x","point_9y","point_10x","point_10y","point_11x","point_11y","point_12x","point_12y","point_13x","point_13y","point_14x","point_14y","point_15x","point_15y","point_16x","point_16y","point_17x","point_17y","point_18x","point_18y","point_19x","point_19y","point_20x","point_20y","point_21x","point_21y","point_22x","point_22y","point_23x","point_23y","point_24x","point_24y","point_25x","point_25y","point_26x","point_26y","point_27x","point_27y","point_28x","point_28y","point_29x","point_29y","point_30x","point_30y","point_31x","point_31y","point_32x","point_32y","point_33x","point_33y", "image" }; private Context context; @@ -175,6 +176,9 @@ public class GameOpenHelper extends SQLiteOpenHelper { db.execSQL(GAME_TABLE_CREATE); db.execSQL(HIT_TABLE_CREATE); } + if(oldVersion == 2 && newVersion == 3) { + db.execSQL("ALTER TABLE hits ADD image BLOB;"); + } } public void insertGame(Game game) { @@ -299,6 +303,11 @@ public class GameOpenHelper extends SQLiteOpenHelper { values.put("remoteId", hit.remoteId); } + byte[] frameBytes = hit.getFrameAsByteArray(); + if(frameBytes != null) { + values.put("image", frameBytes); + } + // Insert the new row, returning the primary key value of the new row long newRowId = db.insert(HIT_TABLE_NAME, null, values); hit.id = newRowId; @@ -344,6 +353,9 @@ public class GameOpenHelper extends SQLiteOpenHelper { ContentValues values = new ContentValues(); values.put("remoteId", hit.remoteId); + // reset image after sync to avoid to much data used on phone +// values.putNull("image"); + // Which row to update, based on the title String selection = "id = ?"; String[] selectionArgs = { Long.toString(hit.id) }; diff --git a/app/src/main/java/com/rubenvandeven/emotionhero/GamingActivity.java b/app/src/main/java/com/rubenvandeven/emotionhero/GamingActivity.java index 393860f..b00ee59 100644 --- a/app/src/main/java/com/rubenvandeven/emotionhero/GamingActivity.java +++ b/app/src/main/java/com/rubenvandeven/emotionhero/GamingActivity.java @@ -6,6 +6,7 @@ import android.app.Dialog; import android.content.DialogInterface; import android.content.Intent; import android.content.pm.PackageManager; +import android.graphics.Typeface; import android.media.AudioAttributes; import android.media.AudioManager; import android.media.SoundPool; @@ -155,6 +156,9 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL StoryDialogFragment storyDialog = new StoryDialogFragment(); storyDialog.show(getSupportFragmentManager(), "StoryDialog"); + Typeface font = Typeface.createFromAsset(getAssets(), "unifont-9.0.02.ttf"); + mContentView.setTypeface(font); + } public void startGame() { @@ -248,6 +252,7 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL Face face = list.get(0); currentScenario.setCurrentFace(face); scenarioView.setCurrentAttributeScoresForFace(face); + currentScenario.currentFrame = (Frame.ByteArrayFrame) frame; } } diff --git a/app/src/main/java/com/rubenvandeven/emotionhero/Hit.java b/app/src/main/java/com/rubenvandeven/emotionhero/Hit.java index a9529b8..4176aab 100644 --- a/app/src/main/java/com/rubenvandeven/emotionhero/Hit.java +++ b/app/src/main/java/com/rubenvandeven/emotionhero/Hit.java @@ -1,12 +1,17 @@ package com.rubenvandeven.emotionhero; import android.database.Cursor; +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; import android.graphics.PointF; +import android.graphics.RectF; +import android.util.Base64; import com.affectiva.android.affdex.sdk.detector.Face; import com.google.gson.annotations.Expose; -import java.util.ArrayList; +import java.io.ByteArrayOutputStream; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -38,10 +43,12 @@ public class Hit { public String remoteId; + public Bitmap frame; + /** * Constructor for game moment */ - public Hit(Scenario.Target target, Game game, Face face, float score) { + public Hit(Scenario.Target target, Game game, Face face, float score, Bitmap frame) { this.score = score; this.target = target; this.game = game; @@ -60,6 +67,10 @@ public class Hit { this.points = face.getFacePoints().clone(); + this.frame = frame; + + // this in a way is strange behaviour: + // preferably do the game.addHit after creating hit, and call hit.game = game from there. game.addHit(this); } @@ -125,6 +136,52 @@ public class Hit { this.points[32] = new PointF(c.getFloat(c.getColumnIndex("point_32x")), c.getFloat(c.getColumnIndex("point_32y"))); this.points[33] = new PointF(c.getFloat(c.getColumnIndex("point_33x")), c.getFloat(c.getColumnIndex("point_33y"))); + byte[] image = c.getBlob(c.getColumnIndex("image")); + if(image != null) + { + frame = BitmapFactory.decodeByteArray(image, 0, image.length); + } + game.addHit(this); } + + public static RectF getBoundingboxForPoints(PointF[] points) { + // a bit of a hack to be as quick as possible + // we know which point is where on the face, however, the face might be tilted: + float[] minXes = {points[0].x,points[1].x,points[5].x}; + float[] maxXes = {points[4].x,points[3].x,points[10].x}; + float[] maxYs = {points[1].y,points[2].y,points[3].y}; + float[] minYs = {points[5].y,points[6].y,points[7].y,points[8].y,points[9].y,points[10].y}; + + Arrays.sort(minXes); + Arrays.sort(maxXes); + Arrays.sort(maxYs); + Arrays.sort(minYs); + + return new RectF(minXes[0], minYs[0], maxXes[maxXes.length-1], maxYs[maxYs.length-1]); + } + + // to store in db + public static String bitmapToBase64(Bitmap bitmap) { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); +// bitmap.compress(Bitmap.CompressFormat.PNG, 100, byteArrayOutputStream); + bitmap.compress(Bitmap.CompressFormat.JPEG, 90, byteArrayOutputStream); + byte[] byteArray = byteArrayOutputStream .toByteArray(); + return Base64.encodeToString(byteArray, Base64.DEFAULT); + } + + // to retreive from db + public static Bitmap base64ToBitmap(String b64) { + byte[] imageAsBytes = Base64.decode(b64.getBytes(), Base64.DEFAULT); + return BitmapFactory.decodeByteArray(imageAsBytes, 0, imageAsBytes.length); + } + + public byte[] getFrameAsByteArray() { + if(frame == null) + return null; + + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + frame.compress(Bitmap.CompressFormat.JPEG, 90, stream); + return stream.toByteArray(); + } } diff --git a/app/src/main/java/com/rubenvandeven/emotionhero/MenuActivity.java b/app/src/main/java/com/rubenvandeven/emotionhero/MenuActivity.java index 3863b41..311f84a 100644 --- a/app/src/main/java/com/rubenvandeven/emotionhero/MenuActivity.java +++ b/app/src/main/java/com/rubenvandeven/emotionhero/MenuActivity.java @@ -15,6 +15,9 @@ import android.widget.Button; import android.widget.ImageView; import android.widget.Toast; +/** + * @deprecated Now done using MirrorMenuActivity + */ public class MenuActivity extends AppCompatActivity { Player player; diff --git a/app/src/main/java/com/rubenvandeven/emotionhero/MirrorMenuActivity.java b/app/src/main/java/com/rubenvandeven/emotionhero/MirrorMenuActivity.java index a08c532..d125330 100644 --- a/app/src/main/java/com/rubenvandeven/emotionhero/MirrorMenuActivity.java +++ b/app/src/main/java/com/rubenvandeven/emotionhero/MirrorMenuActivity.java @@ -6,7 +6,8 @@ import android.animation.ValueAnimator; import android.app.ProgressDialog; import android.content.Intent; import android.content.pm.PackageManager; -import android.graphics.Color; +import android.graphics.Bitmap; +import android.support.v8.renderscript.RenderScript; import android.graphics.Typeface; import android.support.v4.app.ActivityCompat; import android.support.v7.app.AppCompatActivity; @@ -28,10 +29,10 @@ import com.affectiva.android.affdex.sdk.detector.CameraDetector; import com.affectiva.android.affdex.sdk.detector.Detector; import com.affectiva.android.affdex.sdk.detector.Face; -import org.w3c.dom.Text; - import java.util.List; +import io.github.silvaren.easyrs.tools.Nv21Image; + public class MirrorMenuActivity extends AppCompatActivity implements Detector.ImageListener, CameraDetector.CameraEventListener, Detector.FaceListener { final static String LOG_TAG = "EmotionHero-Mirror"; @@ -52,6 +53,8 @@ public class MirrorMenuActivity extends AppCompatActivity implements Detector.Im ProgressDialog loadGameDialog; + RenderScript rs; + @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); @@ -72,6 +75,7 @@ public class MirrorMenuActivity extends AppCompatActivity implements Detector.Im startButton .setTypeface(font); highscoresButton .setTypeface(font); creditsButton .setTypeface(font); + messageText.setTypeface(font); //We create a custom SurfaceView that resizes itself to match the aspect ratio of the incoming camera frames cameraPreview = new SurfaceView(this) { @@ -183,6 +187,8 @@ public class MirrorMenuActivity extends AppCompatActivity implements Detector.Im startActivity(intent); } }); + + rs = RenderScript.create(this); } @Override @@ -227,8 +233,8 @@ public class MirrorMenuActivity extends AppCompatActivity implements Detector.Im detector.setDetectAllEmotions(true); detector.setDetectAllAppearances(false); detector.setDetectAllEmojis(false); - detector.setDetectAllExpressions(true); - detector.setMaxProcessRate(20); + detector.setDetectAllExpressions(false); + detector.setMaxProcessRate(12); detector.setImageListener(this); detector.setOnCameraEventListener(this); @@ -252,10 +258,10 @@ public class MirrorMenuActivity extends AppCompatActivity implements Detector.Im /** * Detector callback gives the faces found so we can match their hits to the given scenario. */ - public void onImageResults(List list, Frame frame, float timestamp) {if (list == null) - return; - if (list.size() == 0) { + public void onImageResults(List list, Frame frame, float timestamp) { + if (list == null || list.size() == 0) { setText("Show me your face"); + return; } else { hideText(); Face face = list.get(0); diff --git a/app/src/main/java/com/rubenvandeven/emotionhero/Scenario.java b/app/src/main/java/com/rubenvandeven/emotionhero/Scenario.java index e7c7e63..e6d5df0 100644 --- a/app/src/main/java/com/rubenvandeven/emotionhero/Scenario.java +++ b/app/src/main/java/com/rubenvandeven/emotionhero/Scenario.java @@ -1,8 +1,14 @@ package com.rubenvandeven.emotionhero; import android.graphics.Bitmap; +import android.graphics.Canvas; +import android.graphics.Color; +import android.graphics.Paint; +import android.graphics.RectF; +import android.support.v8.renderscript.RenderScript; import android.util.Log; +import com.affectiva.android.affdex.sdk.Frame; import com.affectiva.android.affdex.sdk.detector.Face; import java.util.ArrayList; @@ -12,6 +18,8 @@ import java.util.Map; import java.util.Timer; import java.util.TimerTask; +import io.github.silvaren.easyrs.tools.Nv21Image; + /** * Created by ruben on 16/08/16. */ @@ -20,7 +28,7 @@ public class Scenario { public int id; - public Bitmap currentFrameBitmap; + public Frame.ByteArrayFrame currentFrame; public static final int LVL_NONE = 0; public static final int LVL_ANGER = 1; @@ -42,9 +50,16 @@ public class Scenario { /** * Minimum score to be able to pass to the next level. + * (check minimumScore || minimumAchievements) */ float minimumScore = 0; + /** + * Number of achievements required to have, before being able to go to next level + * (check minimumScore || minimumAchievements) + */ + float minimumAchievements= 0; + /** * If a game is beign played. */ @@ -72,6 +87,10 @@ public class Scenario { int maxScore = 0; + protected RenderScript rs; + + protected Paint squarePaint; + /** * The scorres in this moment, as to draw them on the screen. * Indexes are Emotion ordinals @@ -118,6 +137,10 @@ public class Scenario { }; timer.schedule(tickTask, 0, 1000/DESIRED_FPS); this.game = new Game(null, this, 0, new Date(), null); + + rs = RenderScript.create(_activity); + squarePaint = new Paint(); + squarePaint.setColor(Color.YELLOW); } /** @@ -144,13 +167,38 @@ public class Scenario { float scored_value = target.emotion.getValueFromFace(currentFace); float required_value = target.value; float score = 100 - Math.abs(scored_value-required_value); - Hit hit = new Hit(target, game, currentFace, score); - // _activity.sound.play(_activity.soundIds.get(_activity.SOUND_SCORE), 1, 1, - 1,0, hit.score / 200f + 0.5f ); // play back the sound slower + 1,0, score / 200f + 0.5f ); // play back the sound slower // depending on hit value target.isHit = true; + + + + Bitmap outputBitmap = null; + if(currentFrame != null) { + // convert NV21 byteArrayFrame from camera to RGB bitmap. + Frame.ByteArrayFrame byteArrayFrame = (Frame.ByteArrayFrame) currentFrame; + outputBitmap = Nv21Image.nv21ToBitmap(rs, byteArrayFrame.getByteArray(), byteArrayFrame.getWidth(), byteArrayFrame.getHeight()); + + Canvas canvas = new Canvas(outputBitmap); + RectF rect = Hit.getBoundingboxForPoints(currentFace.getFacePoints()); + canvas.drawRect(rect, squarePaint); + + /* + Also possible: only grayscale image - only loops over first part of the bytearray + int[] pixels; + int p; + int size = width*height; + for(int i = 0; i < size; i++) { + p = data[i] & 0xFF; + pixels[i] = 0xff000000 | p<<16 | p<<8 | p; + } + Bitmap bm = Bitmap.createBitmap(pixels, width, height, Bitmap.Config.ARGB_8888); + */ + } + + Hit hit = new Hit(target, game, currentFace, score, outputBitmap); } } diff --git a/app/src/main/res/layout/activity_gaming.xml b/app/src/main/res/layout/activity_gaming.xml index 62cea64..35cdd1d 100644 --- a/app/src/main/res/layout/activity_gaming.xml +++ b/app/src/main/res/layout/activity_gaming.xml @@ -27,11 +27,7 @@ android:textColor="#ffffff" android:textSize="50sp" android:textStyle="bold" - android:paddingBottom="100dp" - android:shadowColor="@android:color/black" - android:shadowDx="0" - android:shadowDy="0" - android:shadowRadius="20" /> + android:paddingBottom="100dp" /> diff --git a/app/src/main/res/layout/activity_review.xml b/app/src/main/res/layout/activity_review.xml index 2445f5c..95cf784 100644 --- a/app/src/main/res/layout/activity_review.xml +++ b/app/src/main/res/layout/activity_review.xml @@ -143,7 +143,7 @@ android:layout_marginBottom="5dp" /> + tools:text="You hit a 50% on the 2nd target, to get a 63% score, raise your brows 2% more." + android:textSize="18sp" />