From 79a8643a04801e2c16b119891d419a5a735aaba4 Mon Sep 17 00:00:00 2001 From: Ruben Date: Thu, 15 Sep 2016 15:23:38 +0100 Subject: [PATCH] Send images to API and add sound fx and upgrade dependencies --- .idea/misc.xml | 2 +- app/build.gradle | 18 +- app/src/main/AndroidManifest.xml | 4 +- .../emotionhero/ApiRestClient.java | 167 +++++++++++++++++- .../emotionhero/GameOpenHelper.java | 21 ++- .../emotionhero/GamingActivity.java | 3 + .../emotionhero/HighscoreActivity.java | 1 + .../emotionhero/MirrorMenuActivity.java | 18 +- .../emotionhero/ProgressActivity.java | 3 + .../emotionhero/ReviewActivity.java | 2 + .../rubenvandeven/emotionhero/Scenario.java | 29 ++- .../main/res/layout/activity_mirror_menu.xml | 2 +- 12 files changed, 240 insertions(+), 30 deletions(-) diff --git a/.idea/misc.xml b/.idea/misc.xml index 1b77328..bd04605 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -37,7 +37,7 @@ - + diff --git a/app/build.gradle b/app/build.gradle index ee60818..119e93c 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -3,13 +3,13 @@ apply plugin: 'com.android.application' android { signingConfigs { } - compileSdkVersion 23 - buildToolsVersion '23.0.1' + compileSdkVersion 24 + buildToolsVersion '23.0.3' defaultConfig { - applicationId "com.rubenvandeven.emotionhero" + applicationId 'com.rubenvandeven.emotion_hero' minSdkVersion 16 - targetSdkVersion 23 - versionCode 1 + targetSdkVersion 24 + versionCode 2 versionName '0.2' testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner" @@ -18,7 +18,7 @@ android { } buildTypes { release { - minifyEnabled true + minifyEnabled false proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro' } } @@ -31,11 +31,11 @@ dependencies { androidTestCompile('com.android.support.test.espresso:espresso-core:2.2.2', { exclude group: 'com.android.support', module: 'support-annotations' }) - compile 'com.android.support:appcompat-v7:23.4.0' - compile 'com.android.support:support-v4:23.4.0' + compile 'com.android.support:appcompat-v7:24.2.0' + compile 'com.android.support:support-v4:24.2.0' compile 'com.affectiva.android:affdexsdk:3.1' compile 'com.google.code.gson:gson:2.4' - compile 'com.android.support:design:23.4.0' + compile 'com.android.support:design:24.2.0' compile 'com.loopj.android:android-async-http:1.4.9' compile 'org.ocpsoft.prettytime:prettytime:4.0.1.Final' compile 'io.github.silvaren:easyrs:0.5.3' diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 15b50da..a7163e0 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -42,11 +42,11 @@ + android:value="com.rubenvandeven.emotionhero.ProgressActivity" /> hits) { + // 1. Cut out the essential parts + GameOpenHelper gameHelper = player.getGameOpenHelper(); + if(hits == null || hits.size() < 1) { + return; + } + + RequestParams params = new RequestParams(); + + boolean noImages = true; + + for(Hit hit: hits) { + Bitmap img = gameHelper.getImageForHit(hit); + if(img == null) { + Log.e("API", "no image for hit " + hit.id); + continue; + } + + noImages = false; + +// Crop: http://stackoverflow.com/a/31698091 + int margin = (int) ((hit.points[10].x - hit.points[5].x) * 0.05); + int left = (int) (hit.points[5].x) - margin; + int right = (int) (hit.points[10].x) + margin; + int top = (int) (hit.points[6].y < hit.points[9].y ? hit.points[6].y : hit.points[9].y) - 2*margin; // a bit of forehead + int bottom = (int) (hit.points[5].y > hit.points[10].y ? hit.points[5].y : hit.points[10].y) + margin; + Rect rect = new Rect(left, top, right, bottom); +// Be sure that there is at least 1px to slice. + if(!(rect.left < rect.right && rect.top < rect.bottom)) { + Log.e("API", "Error in point positions."+" left: " + rect.left + " right: " + rect.right + " top: " + rect.top + " bottom: " + rect.bottom ); + continue; // strange bug... skip it and drop the file anyway + } +// Create our resulting image (150--50),(75--25) = 200x100px + Bitmap croppedBmp = Bitmap.createBitmap(rect.right-rect.left, rect.bottom-rect.top, Bitmap.Config.ARGB_8888); +// draw source bitmap into resulting image at given position: + new Canvas(croppedBmp).drawBitmap(img, -rect.left, -rect.top, null); + + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + croppedBmp.compress(Bitmap.CompressFormat.JPEG, 90, stream); + byte[] imageBytes = stream.toByteArray(); + + params.put(hit.remoteId + ":brows", new ByteArrayInputStream(imageBytes), hit.remoteId + "-brows.jpg", "image/jpeg"); + Log.v("API", "add param: " + hit.remoteId + ":brows - length:" + imageBytes.length ); + +// NOSE +// Crop: http://stackoverflow.com/a/31698091 + // reuse margin of brows + left = (int) hit.points[13].x - margin; + right = (int) hit.points[15].x + margin; + top = (int) hit.points[11].y - margin; + bottom = (int) hit.points[14].y + margin; + Rect rect2 = new Rect(left, top, right, bottom); +// Be sure that there is at least 1px to slice. + if(!(rect2.left < rect2.right && rect2.top < rect2.bottom)) { + Log.e("API", "Error in point positions."+" left: " + rect2.left + " right: " + rect2.right + " top: " + rect2.top + " bottom: " + rect2.bottom ); + continue; // strange bug... skip it and drop the file anyway + } +// Create our resulting image (150--50),(75--25) = 200x100px + croppedBmp = Bitmap.createBitmap(rect2.right-rect2.left, rect2.bottom-rect2.top, Bitmap.Config.ARGB_8888); +// draw source bitmap into resulting image at given position: + new Canvas(croppedBmp).drawBitmap(img, -rect2.left, -rect2.top, null); + + ByteArrayOutputStream stream2 = new ByteArrayOutputStream(); + croppedBmp.compress(Bitmap.CompressFormat.JPEG, 90, stream2); + byte[] imageBytes2 = stream2.toByteArray(); + + params.put(hit.remoteId + ":nose", new ByteArrayInputStream(imageBytes2), hit.remoteId + "-nose.jpg", "image/jpeg"); + Log.v("API", "add param: " + hit.remoteId + ":nose - length:" + imageBytes.length ); + +// RIGHT MOUTH CORNER +// Crop: http://stackoverflow.com/a/31698091 + // reuse margin of brows to make a square image + left = (int) hit.points[24].x - margin; + right = (int) hit.points[24].x + 3 * margin; + top = (int) hit.points[24].y - 2 * margin; + bottom = (int) hit.points[24].y + 2 * margin; + Rect rect3 = new Rect(left, top, right, bottom); +// Be sure that there is at least 1px to slice. + if(!(rect3.left < rect3.right && rect3.top < rect3.bottom)) { + Log.e("API", "Error in point positions."+" left: " + rect3.left + " right: " + rect3.right + " top: " + rect3.top + " bottom: " + rect3.bottom ); + continue; // strange bug... skip it and drop the file anyway + } + + croppedBmp = Bitmap.createBitmap(rect3.right-rect3.left, rect3.bottom-rect3.top, Bitmap.Config.ARGB_8888); +// draw source bitmap into resulting image at given position: + new Canvas(croppedBmp).drawBitmap(img, -rect3.left, -rect3.top, null); + + ByteArrayOutputStream stream3 = new ByteArrayOutputStream(); + croppedBmp.compress(Bitmap.CompressFormat.JPEG, 90, stream3); + byte[] imageBytes3 = stream3.toByteArray(); + + params.put(hit.remoteId + ":mouth_right", new ByteArrayInputStream(imageBytes3), hit.remoteId + "-mouth_right.jpg", "image/jpeg"); + Log.v("API", "add param: " + hit.remoteId + ":mouth_right - length:" + imageBytes.length ); + } + + // don't post if there are no images + // this happens ie. when images are all successfully send + if(noImages) { + return; + } + + post("/images", params,new JsonHttpResponseHandler(){ + @Override + public void onSuccess(int statusCode, Header[] headers, JSONObject response) { +// 3. remove images :-) + for(Hit hit: hits) { + player.getGameOpenHelper().clearImageForHit(hit); + } + } + + @Override + public void onFailure(int statusCode, Header[] headers, String responseString, Throwable throwable) { + handleOnFailure(statusCode, headers, responseString, throwable); + } + + @Override + public void onFailure(int statusCode, Header[] headers, Throwable throwable, JSONObject response) { + onFailure(statusCode, headers, response == null ? null:response.toString(), throwable); + } + }); + } + public static void handleOnFailure(int statusCode, Header[] headers, String responseString, Throwable throwable) { Log.e("API", "FAILURE ON REQUEST!"); Log.e("API", throwable.getMessage()); diff --git a/app/src/main/java/com/rubenvandeven/emotionhero/GameOpenHelper.java b/app/src/main/java/com/rubenvandeven/emotionhero/GameOpenHelper.java index d5de0de..a4ff100 100644 --- a/app/src/main/java/com/rubenvandeven/emotionhero/GameOpenHelper.java +++ b/app/src/main/java/com/rubenvandeven/emotionhero/GameOpenHelper.java @@ -6,6 +6,8 @@ import android.content.Intent; import android.database.Cursor; import android.database.sqlite.SQLiteDatabase; import android.database.sqlite.SQLiteOpenHelper; +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; import android.graphics.PointF; import android.util.Log; @@ -153,7 +155,7 @@ public class GameOpenHelper extends SQLiteOpenHelper { "image BLOB," + "PRIMARY KEY (id));" ; private static final String[] HIT_PROJECTION = { - "id","game_id", "target_index","score","bonus","remoteId","glasses","ethnicity","age","gender","anger","contempt","disgust","fear","joy","sadness","surprise","roll","pitch","yaw","inter_ocular_distance","mouth_open","lip_press","brow_raise","nose_wrinkler","lip_depressor","brow_furrow","attention","smile","inner_brow_raiser","chin_raiser","smirk","lip_suck","upper_lip_raiser","lip_pucker","eye_closure","engagement","valence","point_0x","point_0y","point_1x","point_1y","point_2x","point_2y","point_3x","point_3y","point_4x","point_4y","point_5x","point_5y","point_6x","point_6y","point_7x","point_7y","point_8x","point_8y","point_9x","point_9y","point_10x","point_10y","point_11x","point_11y","point_12x","point_12y","point_13x","point_13y","point_14x","point_14y","point_15x","point_15y","point_16x","point_16y","point_17x","point_17y","point_18x","point_18y","point_19x","point_19y","point_20x","point_20y","point_21x","point_21y","point_22x","point_22y","point_23x","point_23y","point_24x","point_24y","point_25x","point_25y","point_26x","point_26y","point_27x","point_27y","point_28x","point_28y","point_29x","point_29y","point_30x","point_30y","point_31x","point_31y","point_32x","point_32y","point_33x","point_33y", "image" + "id","game_id", "target_index","score","bonus","remoteId","glasses","ethnicity","age","gender","anger","contempt","disgust","fear","joy","sadness","surprise","roll","pitch","yaw","inter_ocular_distance","mouth_open","lip_press","brow_raise","nose_wrinkler","lip_depressor","brow_furrow","attention","smile","inner_brow_raiser","chin_raiser","smirk","lip_suck","upper_lip_raiser","lip_pucker","eye_closure","engagement","valence","point_0x","point_0y","point_1x","point_1y","point_2x","point_2y","point_3x","point_3y","point_4x","point_4y","point_5x","point_5y","point_6x","point_6y","point_7x","point_7y","point_8x","point_8y","point_9x","point_9y","point_10x","point_10y","point_11x","point_11y","point_12x","point_12y","point_13x","point_13y","point_14x","point_14y","point_15x","point_15y","point_16x","point_16y","point_17x","point_17y","point_18x","point_18y","point_19x","point_19y","point_20x","point_20y","point_21x","point_21y","point_22x","point_22y","point_23x","point_23y","point_24x","point_24y","point_25x","point_25y","point_26x","point_26y","point_27x","point_27y","point_28x","point_28y","point_29x","point_29y","point_30x","point_30y","point_31x","point_31y","point_32x","point_32y","point_33x","point_33y" }; private static final String GAME_ACH_TABLE_NAME = "game_achievements"; @@ -207,6 +209,23 @@ public class GameOpenHelper extends SQLiteOpenHelper { } + public Bitmap getImageForHit(Hit hit) { + String[] params = { Long.toString(hit.id) }; + Cursor c = getReadableDatabase().rawQuery("SELECT image FROM hits h WHERE h.id = ?", params); + c.moveToFirst(); + byte[] imageBytes= c.getBlob(0); + if(imageBytes == null) + { + return null; + } + return BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.length); + } + + public void clearImageForHit(Hit hit) { + String[] params = { Long.toString(hit.id) }; + getWritableDatabase().rawQuery("UPDATE hits SET image = NULL WHERE id = ?", params); + } + public void insertGame(Game game) { // Gets the data repository in write mode SQLiteDatabase db = getWritableDatabase(); diff --git a/app/src/main/java/com/rubenvandeven/emotionhero/GamingActivity.java b/app/src/main/java/com/rubenvandeven/emotionhero/GamingActivity.java index a2d3a92..8074b9c 100644 --- a/app/src/main/java/com/rubenvandeven/emotionhero/GamingActivity.java +++ b/app/src/main/java/com/rubenvandeven/emotionhero/GamingActivity.java @@ -65,6 +65,7 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL public HashMap soundIds = new HashMap<>(); final static int SOUND_SCORE = 1; + final static int SOUND_BONUS = 2; protected Player player; @@ -154,6 +155,7 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL createSoundPool(); // instantiate SoundPool in sound soundIds.put(SOUND_SCORE, sound.load(this, R.raw.score2, 1)); + soundIds.put(SOUND_BONUS, sound.load(this, R.raw.scorebonus, 1)); // StoryDialogFragment storyDialog = new StoryDialogFragment(); // storyDialog.show(getSupportFragmentManager(), "StoryDialog"); @@ -345,6 +347,7 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL .setContentType(AudioAttributes.CONTENT_TYPE_SONIFICATION) .build(); sound = new SoundPool.Builder() + .setMaxStreams(6) // allow sounds to ring while another is started .setAudioAttributes(attributes) .build(); } diff --git a/app/src/main/java/com/rubenvandeven/emotionhero/HighscoreActivity.java b/app/src/main/java/com/rubenvandeven/emotionhero/HighscoreActivity.java index 0513a2e..f896c00 100644 --- a/app/src/main/java/com/rubenvandeven/emotionhero/HighscoreActivity.java +++ b/app/src/main/java/com/rubenvandeven/emotionhero/HighscoreActivity.java @@ -291,6 +291,7 @@ public class HighscoreActivity extends AppCompatActivity { startLvlButton.setVisibility(View.INVISIBLE); } + nextLvlButton.setVisibility(View.GONE); if(scenario.isFinalLevel()) { nextLvlButton.setVisibility(View.INVISIBLE); } else { diff --git a/app/src/main/java/com/rubenvandeven/emotionhero/MirrorMenuActivity.java b/app/src/main/java/com/rubenvandeven/emotionhero/MirrorMenuActivity.java index ac683ef..3240dd3 100644 --- a/app/src/main/java/com/rubenvandeven/emotionhero/MirrorMenuActivity.java +++ b/app/src/main/java/com/rubenvandeven/emotionhero/MirrorMenuActivity.java @@ -165,14 +165,14 @@ public class MirrorMenuActivity extends AppCompatActivity implements Detector.Im startButton.setOnClickListener(new View.OnClickListener() { @Override public void onClick(View v) { - Intent intent; - if(!player.getPlayerInfo().canContinueLevel()) { - intent = new Intent(MirrorMenuActivity.this, IntroductionActivity.class); - } else { - loadGameDialog = ProgressDialog.show(MirrorMenuActivity.this, "", - MirrorMenuActivity.this.getResources().getString(R.string.load_game_activity), true); - intent = new Intent(MirrorMenuActivity.this, GamingActivity.class); - } + Intent intent = new Intent(MirrorMenuActivity.this, IntroductionActivity.class); +// if(!player.getPlayerInfo().canContinueLevel()) { +// intent = new Intent(MirrorMenuActivity.this, IntroductionActivity.class); +// } else { +// loadGameDialog = ProgressDialog.show(MirrorMenuActivity.this, "", +// MirrorMenuActivity.this.getResources().getString(R.string.load_game_activity), true); +// intent = new Intent(MirrorMenuActivity.this, GamingActivity.class); +// } startActivity(intent); detector.stop(); detector = null; @@ -181,6 +181,8 @@ public class MirrorMenuActivity extends AppCompatActivity implements Detector.Im }); + // merge with 'start button' + if(!player.getPlayerInfo().canContinueLevel()) { highscoresButton.setVisibility(View.GONE); } else { diff --git a/app/src/main/java/com/rubenvandeven/emotionhero/ProgressActivity.java b/app/src/main/java/com/rubenvandeven/emotionhero/ProgressActivity.java index 3cd1b72..0c45f13 100644 --- a/app/src/main/java/com/rubenvandeven/emotionhero/ProgressActivity.java +++ b/app/src/main/java/com/rubenvandeven/emotionhero/ProgressActivity.java @@ -1,5 +1,6 @@ package com.rubenvandeven.emotionhero; +import android.app.ProgressDialog; import android.content.Intent; import android.graphics.Typeface; import android.support.v7.app.AppCompatActivity; @@ -130,6 +131,8 @@ public class ProgressActivity extends AppCompatActivity { intent.putExtra(HighscoreActivity.INTENT_EXTRA_LVL_ID, scenario.id); } else { // "PLAY NOW!" + ProgressDialog dialog = ProgressDialog.show(getApplicationContext(), "", + getApplicationContext().getResources().getString(R.string.load_game_activity), true); intent = new Intent(ProgressActivity.this, GamingActivity.class); intent.putExtra(GamingActivity.INTENT_EXTRA_SCENARIO, scenario.id); } diff --git a/app/src/main/java/com/rubenvandeven/emotionhero/ReviewActivity.java b/app/src/main/java/com/rubenvandeven/emotionhero/ReviewActivity.java index 6e91e42..9da8763 100644 --- a/app/src/main/java/com/rubenvandeven/emotionhero/ReviewActivity.java +++ b/app/src/main/java/com/rubenvandeven/emotionhero/ReviewActivity.java @@ -103,6 +103,8 @@ public class ReviewActivity extends AppCompatActivity { scoreProgressBar.setVisibility(View.GONE); } }); + // try submitting images if not yet done. + player.api.sendHitImages(game.hits.values()); } } }; diff --git a/app/src/main/java/com/rubenvandeven/emotionhero/Scenario.java b/app/src/main/java/com/rubenvandeven/emotionhero/Scenario.java index d017d03..b101a1a 100644 --- a/app/src/main/java/com/rubenvandeven/emotionhero/Scenario.java +++ b/app/src/main/java/com/rubenvandeven/emotionhero/Scenario.java @@ -3,6 +3,7 @@ package com.rubenvandeven.emotionhero; import android.graphics.Bitmap; import android.graphics.Canvas; import android.graphics.Color; +import android.graphics.Matrix; import android.graphics.Paint; import android.graphics.RectF; import android.support.v8.renderscript.RenderScript; @@ -172,8 +173,14 @@ public class Scenario { Hit hit = new Hit(target, game, currentFace); _activity.sound.play(_activity.soundIds.get(_activity.SOUND_SCORE), 1, 1, - 1,0, hit.score / 200f + 0.5f ); // play back the sound slower + 2,0, hit.score / 200f + 0.5f ); // play back the sound slower // depending on hit value + + if(hit.bonus > 0) { + _activity.sound.play(_activity.soundIds.get(_activity.SOUND_BONUS), 0.7f, 0.7f, + 1,0, hit.bonus / 100f + 0.8f ); // play back the sound slower + // depending on bonus value + } target.isHit = true; @@ -183,13 +190,25 @@ public class Scenario { // convert NV21 byteArrayFrame from camera to RGB bitmap. Frame.ByteArrayFrame byteArrayFrame = (Frame.ByteArrayFrame) currentFrame; outputBitmap = Nv21Image.nv21ToBitmap(rs, byteArrayFrame.getByteArray(), byteArrayFrame.getWidth(), byteArrayFrame.getHeight()); + Frame.ROTATE rotation = byteArrayFrame.getTargetRotation(); +// ie BY_90_CCW -90.0 + Log.v("Scenario", "frame rotation: " + rotation.toString() + " " + rotation.toDouble()); + // process rotation... (maybe faster if done on byte array?) + if(rotation != Frame.ROTATE.NO_ROTATION) { + Matrix matrix = new Matrix(); + matrix.postRotate((int) rotation.toDouble()); +// int width = rotation == Frame.ROTATE.BY_180 ? outputBitmap.getWidth() : outputBitmap.getHeight(); +// int height = rotation == Frame.ROTATE.BY_180 ? outputBitmap.getHeight() : outputBitmap.getWidth(); + outputBitmap = Bitmap.createBitmap(outputBitmap , 0, 0, outputBitmap .getWidth(), outputBitmap .getHeight(), matrix, true); + } - Canvas canvas = new Canvas(outputBitmap); - RectF rect = Hit.getBoundingboxForPoints(currentFace.getFacePoints()); - canvas.drawRect(rect, squarePaint); +// Deprecated: now we send only fragments of the images anyway. +// Canvas canvas = new Canvas(outputBitmap); +// RectF rect = Hit.getBoundingboxForPoints(currentFace.getFacePoints()); +// canvas.drawRect(rect, squarePaint); /* - Also possible: only grayscale image - only loops over first part of the bytearray + Also possible: only grayscale image - only loops over first part of the bytearray, is it faster? int[] pixels; int p; int size = width*height; diff --git a/app/src/main/res/layout/activity_mirror_menu.xml b/app/src/main/res/layout/activity_mirror_menu.xml index e47fe7b..34c75e4 100644 --- a/app/src/main/res/layout/activity_mirror_menu.xml +++ b/app/src/main/res/layout/activity_mirror_menu.xml @@ -40,7 +40,7 @@ android:textColor="@color/textPrimary" android:textAllCaps="true" android:layout_marginBottom="@dimen/fab_margin" - android:text="@string/highscores" + android:text="@string/continueBtn" android:layout_width="match_parent" android:layout_height="wrap_content" android:id="@+id/highscoresButton"