From e8b4a03c9d54af6caac847de4aa82272b907aacb Mon Sep 17 00:00:00 2001 From: Ruben Date: Mon, 19 Sep 2016 01:12:38 +0100 Subject: [PATCH] Add extra expression parameters, extra image creation and small fixes, and android version bump --- app/build.gradle | 6 +-- .../emotionhero/ApiRestClient.java | 39 +++++++++++++++++++ .../rubenvandeven/emotionhero/Expression.java | 22 ++++++++++- .../emotionhero/GameOpenHelper.java | 26 ++++++++++--- .../com/rubenvandeven/emotionhero/Hit.java | 2 +- .../rubenvandeven/emotionhero/Scenario.java | 2 +- 6 files changed, 85 insertions(+), 12 deletions(-) diff --git a/app/build.gradle b/app/build.gradle index 0f4029f..26242b9 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -9,8 +9,8 @@ android { applicationId 'com.rubenvandeven.emotion_hero' minSdkVersion 16 targetSdkVersion 24 - versionCode 4 - versionName '0.4' + versionCode 6 + versionName '0.5' testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner" renderscriptTargetApi 16 @@ -33,7 +33,7 @@ dependencies { }) compile 'com.android.support:appcompat-v7:24.2.0' compile 'com.android.support:support-v4:24.2.0' - compile 'com.affectiva.android:affdexsdk:3.1' + compile 'com.affectiva.android:affdexsdk:3.1.2' compile 'com.google.code.gson:gson:2.4' compile 'com.android.support:design:24.2.0' compile 'com.loopj.android:android-async-http:1.4.9' diff --git a/app/src/main/java/com/rubenvandeven/emotionhero/ApiRestClient.java b/app/src/main/java/com/rubenvandeven/emotionhero/ApiRestClient.java index c050baa..02b5228 100644 --- a/app/src/main/java/com/rubenvandeven/emotionhero/ApiRestClient.java +++ b/app/src/main/java/com/rubenvandeven/emotionhero/ApiRestClient.java @@ -76,6 +76,7 @@ public class ApiRestClient { */ public void requestWithJWT(final String method, final String url, final StringEntity postBody, final RequestParams getParams, final AsyncHttpResponseHandler responseHandler) { // sync call, so we can return a value! + Log.v("API", "register"); client.post(BASE_URL + "/api/register", null, new JsonHttpResponseHandler() { @Override public void onSuccess(int statusCode, Header[] headers, JSONObject response) { @@ -99,6 +100,16 @@ public class ApiRestClient { e.printStackTrace(); } } + + @Override + public void onFailure(int statusCode, Header[] headers, String responseString, Throwable throwable) { + handleOnFailure(statusCode, headers, responseString, throwable); + } + + @Override + public void onFailure(int statusCode, Header[] headers, Throwable throwable, JSONObject response) { + onFailure(statusCode, headers, response == null ? null:response.toString(), throwable); + } }); } @@ -335,6 +346,7 @@ public class ApiRestClient { for(Hit hit: hits) { Bitmap img = gameHelper.getImageForHit(hit); + if(img == null) { Log.e("API", "no image for hit " + hit.id); continue; @@ -415,6 +427,32 @@ public class ApiRestClient { params.put(hit.remoteId + ":mouth_right", new ByteArrayInputStream(imageBytes3), hit.remoteId + "-mouth_right.jpg", "image/jpeg"); Log.v("API", "add param: " + hit.remoteId + ":mouth_right - length:" + imageBytes.length ); + + +// LEFT MOUTH CORNER (doing whole mouth :-) +// Crop: http://stackoverflow.com/a/31698091 + // reuse margin of brows to make a square image + left = (int) hit.points[20].x - margin; + right = (int) hit.points[24].x + margin; + top = (int) (hit.points[21].y < hit.points[23].y ? hit.points[21].y : hit.points[23].y) - margin; + bottom = (int) hit.points[26].y + margin; + Rect rect4 = new Rect(left, top, right, bottom); +// Be sure that there is at least 1px to slice. + if(!(rect4.left < rect4.right && rect4.top < rect4.bottom)) { + Log.e("API", "Error in point positions."+" left: " + rect4.left + " right: " + rect4.right + " top: " + rect4.top + " bottom: " + rect4.bottom ); + continue; // strange bug... skip it and drop the file anyway + } + + croppedBmp = Bitmap.createBitmap(rect4.right-rect4.left, rect4.bottom-rect4.top, Bitmap.Config.ARGB_8888); +// draw source bitmap into resulting image at given position: + new Canvas(croppedBmp).drawBitmap(img, -rect4.left, -rect4.top, null); + + ByteArrayOutputStream stream4 = new ByteArrayOutputStream(); + croppedBmp.compress(Bitmap.CompressFormat.JPEG, 90, stream4); + byte[] imageBytes4 = stream4.toByteArray(); + + params.put(hit.remoteId + ":mouth_left", new ByteArrayInputStream(imageBytes4), hit.remoteId + "-mouth_left.jpg", "image/jpeg"); + Log.v("API", "add param: " + hit.remoteId + ":mouth_left - length:" + imageBytes.length ); } // don't post if there are no images @@ -428,6 +466,7 @@ public class ApiRestClient { public void onSuccess(int statusCode, Header[] headers, JSONObject response) { // 3. remove images :-) for(Hit hit: hits) { + Log.v("API", "clear image for hit: "+ hit.id); player.getGameOpenHelper().clearImageForHit(hit); } } diff --git a/app/src/main/java/com/rubenvandeven/emotionhero/Expression.java b/app/src/main/java/com/rubenvandeven/emotionhero/Expression.java index 72d6499..94e0ec9 100644 --- a/app/src/main/java/com/rubenvandeven/emotionhero/Expression.java +++ b/app/src/main/java/com/rubenvandeven/emotionhero/Expression.java @@ -17,7 +17,7 @@ public enum Expression { LIP_PRESS, BROW_RAISE, NOSE_WRINKLER, - LIP_DEPRESSOR, + LIP_DEPRESSOR, //lip corner depressor BROW_FURROW, ATTENTION, SMILE, @@ -29,7 +29,13 @@ public enum Expression { LIP_PUCKER, EYE_CLOSURE, ENGAGEMENT, - VALENCE; + VALENCE, + DIMPLER, + CHEEK_RAISE, + EYE_WIDEN, + JAW_DROP, + LID_TIGHTEN, + LIP_STRETCH; public float getValueFromFace(Face face) { @@ -79,6 +85,18 @@ public enum Expression { return face.emotions.getEngagement(); case VALENCE: return face.emotions.getValence(); + case DIMPLER: + return face.expressions.getDimpler(); + case CHEEK_RAISE: + return face.expressions.getCheekRaise(); + case EYE_WIDEN: + return face.expressions.getEyeWiden(); + case JAW_DROP: + return face.expressions.getJawDrop(); + case LID_TIGHTEN: + return face.expressions.getLidTighten(); + case LIP_STRETCH: + return face.expressions.getLipStretch(); } return 0; } diff --git a/app/src/main/java/com/rubenvandeven/emotionhero/GameOpenHelper.java b/app/src/main/java/com/rubenvandeven/emotionhero/GameOpenHelper.java index e21196e..cd0c0ba 100644 --- a/app/src/main/java/com/rubenvandeven/emotionhero/GameOpenHelper.java +++ b/app/src/main/java/com/rubenvandeven/emotionhero/GameOpenHelper.java @@ -23,7 +23,7 @@ import java.util.Date; * adb pull /sdcard/emotionhero.db */ public class GameOpenHelper extends SQLiteOpenHelper { - private static final int DATABASE_VERSION = 6; + private static final int DATABASE_VERSION = 7; private static final String GAME_TABLE_NAME = "games"; private static final String GAME_TABLE_CREATE = "CREATE TABLE " + GAME_TABLE_NAME + " (" + @@ -86,6 +86,12 @@ public class GameOpenHelper extends SQLiteOpenHelper { "eye_closure VARCHAR(20)," + "engagement VARCHAR(20)," + "valence VARCHAR(20)," + + "dimpler VARCHAR(20)," + + "cheek_raise VARCHAR(20)," + + "eye_widen VARCHAR(20)," + + "jaw_drop VARCHAR(20)," + + "lid_tighten VARCHAR(20)," + + "lip_stretch VARCHAR(20)," + "point_0x VARCHAR(20)," + "point_0y VARCHAR(20)," + "point_1x VARCHAR(20)," + @@ -157,7 +163,7 @@ public class GameOpenHelper extends SQLiteOpenHelper { "image BLOB," + "PRIMARY KEY (id));" ; private static final String[] HIT_PROJECTION = { - "id","game_id", "target_index","score","bonus","remoteId","glasses","ethnicity","age","gender","anger","contempt","disgust","fear","joy","sadness","surprise","roll","pitch","yaw","inter_ocular_distance","mouth_open","lip_press","brow_raise","nose_wrinkler","lip_depressor","brow_furrow","attention","smile","inner_brow_raiser","chin_raiser","smirk","lip_suck","upper_lip_raiser","lip_pucker","eye_closure","engagement","valence","point_0x","point_0y","point_1x","point_1y","point_2x","point_2y","point_3x","point_3y","point_4x","point_4y","point_5x","point_5y","point_6x","point_6y","point_7x","point_7y","point_8x","point_8y","point_9x","point_9y","point_10x","point_10y","point_11x","point_11y","point_12x","point_12y","point_13x","point_13y","point_14x","point_14y","point_15x","point_15y","point_16x","point_16y","point_17x","point_17y","point_18x","point_18y","point_19x","point_19y","point_20x","point_20y","point_21x","point_21y","point_22x","point_22y","point_23x","point_23y","point_24x","point_24y","point_25x","point_25y","point_26x","point_26y","point_27x","point_27y","point_28x","point_28y","point_29x","point_29y","point_30x","point_30y","point_31x","point_31y","point_32x","point_32y","point_33x","point_33y" + "id","game_id", "target_index","score","bonus","remoteId","glasses","ethnicity","age","gender","anger","contempt","disgust","fear","joy","sadness","surprise","roll","pitch","yaw","inter_ocular_distance","mouth_open","lip_press","brow_raise","nose_wrinkler","lip_depressor","brow_furrow","attention","smile","inner_brow_raiser","chin_raiser","smirk","lip_suck","upper_lip_raiser","lip_pucker","eye_closure","engagement","valence","dimpler","cheek_raise","eye_widen","jaw_drop","lid_tighten","lip_stretch","point_0x","point_0y","point_1x","point_1y","point_2x","point_2y","point_3x","point_3y","point_4x","point_4y","point_5x","point_5y","point_6x","point_6y","point_7x","point_7y","point_8x","point_8y","point_9x","point_9y","point_10x","point_10y","point_11x","point_11y","point_12x","point_12y","point_13x","point_13y","point_14x","point_14y","point_15x","point_15y","point_16x","point_16y","point_17x","point_17y","point_18x","point_18y","point_19x","point_19y","point_20x","point_20y","point_21x","point_21y","point_22x","point_22y","point_23x","point_23y","point_24x","point_24y","point_25x","point_25y","point_26x","point_26y","point_27x","point_27y","point_28x","point_28y","point_29x","point_29y","point_30x","point_30y","point_31x","point_31y","point_32x","point_32y","point_33x","point_33y" }; private static final String GAME_ACH_TABLE_NAME = "game_achievements"; @@ -211,6 +217,14 @@ public class GameOpenHelper extends SQLiteOpenHelper { if(oldVersion == 5 && newVersion == 6) { db.execSQL("ALTER TABLE games ADD lost_face_time VARCHAR(255);"); } + if(oldVersion == 6 && newVersion == 7) { + db.execSQL("ALTER TABLE games ADD dimpler VARCHAR(20);"); + db.execSQL("ALTER TABLE games ADD cheek_raise VARCHAR(20);" ); + db.execSQL("ALTER TABLE games ADD eye_widen VARCHAR(20);"); + db.execSQL("ALTER TABLE games ADD jaw_drop VARCHAR(20);"); + db.execSQL("ALTER TABLE games ADD lid_tighten VARCHAR(20);"); + db.execSQL("ALTER TABLE games ADD lip_stretch VARCHAR(20);"); + } } @@ -228,7 +242,9 @@ public class GameOpenHelper extends SQLiteOpenHelper { public void clearImageForHit(Hit hit) { String[] params = { Long.toString(hit.id) }; - getWritableDatabase().rawQuery("UPDATE hits SET image = NULL WHERE id = ?", params); + Cursor c = getWritableDatabase().rawQuery("UPDATE hits SET image = NULL WHERE id = ?", params); + Log.v("GAME", "clear "+hit.id+" count: "+c.getCount()); + } public void insertGame(Game game) { @@ -511,8 +527,8 @@ public class GameOpenHelper extends SQLiteOpenHelper { public int getLocalRankOfGame(Game game) { String[] params = { Long.toString(game.scenario.id), Double.toString(game.score + game.bonus) }; - Log.e("GAME",params[0].toString()); - Log.e("GAME",params[1].toString()); +// Log.e("GAME",params[0].toString()); +// Log.e("GAME",params[1].toString()); Cursor c = getReadableDatabase().rawQuery("SELECT COUNT(id)+1 FROM games WHERE lvl_id = ? AND score+bonus > ?", params); c.moveToFirst(); return c.getInt(0); diff --git a/app/src/main/java/com/rubenvandeven/emotionhero/Hit.java b/app/src/main/java/com/rubenvandeven/emotionhero/Hit.java index 94e27eb..b5ca027 100644 --- a/app/src/main/java/com/rubenvandeven/emotionhero/Hit.java +++ b/app/src/main/java/com/rubenvandeven/emotionhero/Hit.java @@ -39,7 +39,7 @@ public class Hit { public String age; public String gender; - public Map expressions = new HashMap<>(21); // also includes measurements + public Map expressions = new HashMap<>(Expression.values().length); // also includes measurements public PointF[] points = new PointF[34]; diff --git a/app/src/main/java/com/rubenvandeven/emotionhero/Scenario.java b/app/src/main/java/com/rubenvandeven/emotionhero/Scenario.java index 624be62..4a9e382 100644 --- a/app/src/main/java/com/rubenvandeven/emotionhero/Scenario.java +++ b/app/src/main/java/com/rubenvandeven/emotionhero/Scenario.java @@ -170,7 +170,7 @@ public class Scenario { /** * To be called on each timer tick */ - public void tick() + public synchronized void tick() { if(!isRunning){ if(lostFace) {