r
/
emotionhero
1
0
Fork 0
Code Issues 1 Pull Requests Releases Wiki Activity

Send images to API and add sound fx and upgrade dependencies

This commit is contained in:
Ruben 2016-09-15 15:23:38 +01:00
parent b6d9c30560
commit 79a8643a04
12 changed files with 240 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.idea/misc.xml
View File

@ -37,7 +37,7 @@
<ConfirmationsSetting value="0" id="Add" />
<ConfirmationsSetting value="0" id="Remove" />
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" default="true" assert-keyword="true" jdk-15="true" project-jdk-name="1.8" project-jdk-type="JavaSDK">
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_7" default="true" assert-keyword="true" jdk-15="true" project-jdk-name="1.8" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/build/classes" />
</component>
<component name="ProjectType">

18
app/build.gradle
View File

@ -3,13 +3,13 @@ apply plugin: 'com.android.application'
android {
signingConfigs {
}
compileSdkVersion 23
buildToolsVersion '23.0.1'
compileSdkVersion 24
buildToolsVersion '23.0.3'
defaultConfig {
applicationId "com.rubenvandeven.emotionhero"
applicationId 'com.rubenvandeven.emotion_hero'
minSdkVersion 16
targetSdkVersion 23
versionCode 1
targetSdkVersion 24
versionCode 2
versionName '0.2'
testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner"
@ -18,7 +18,7 @@ android {
}
buildTypes {
release {
minifyEnabled true
minifyEnabled false
proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'
}
}
@ -31,11 +31,11 @@ dependencies {
androidTestCompile('com.android.support.test.espresso:espresso-core:2.2.2', {
exclude group: 'com.android.support', module: 'support-annotations'
})
compile 'com.android.support:appcompat-v7:23.4.0'
compile 'com.android.support:support-v4:23.4.0'
compile 'com.android.support:appcompat-v7:24.2.0'
compile 'com.android.support:support-v4:24.2.0'
compile 'com.affectiva.android:affdexsdk:3.1'
compile 'com.google.code.gson:gson:2.4'
compile 'com.android.support:design:23.4.0'
compile 'com.android.support:design:24.2.0'
compile 'com.loopj.android:android-async-http:1.4.9'
compile 'org.ocpsoft.prettytime:prettytime:4.0.1.Final'
compile 'io.github.silvaren:easyrs:0.5.3'

4
app/src/main/AndroidManifest.xml
View File

@ -42,11 +42,11 @@
<activity
android:name=".HighscoreActivity"
android:label="@string/title_activity_highscore"
android:parentActivityName=".MirrorMenuActivity"
android:parentActivityName=".ProgressActivity"
android:theme="@style/AppTheme.NoActionBar">
<meta-data
android:name="android.support.PARENT_ACTIVITY"
android:value="com.rubenvandeven.emotionhero.MirrorMenuActivity" />
android:value="com.rubenvandeven.emotionhero.ProgressActivity" />
</activity>
<activity
android:name=".CreditsActivity"

167
app/src/main/java/com/rubenvandeven/emotionhero/ApiRestClient.java
View File

@ -1,7 +1,10 @@
package com.rubenvandeven.emotionhero;
import android.content.Context;
import android.graphics.Bitmap;
import android.graphics.Canvas;
import android.graphics.PointF;
import android.graphics.Rect;
import android.util.Log;
import com.google.gson.Gson;
@ -16,10 +19,13 @@ import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.UnsupportedEncodingException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Locale;
import java.util.TimeZone;
@ -133,13 +139,18 @@ public class ApiRestClient {
}
}
public void request(String method, String url, StringEntity postBody, RequestParams getParams, AsyncHttpResponseHandler responseHandler) {
public void request(String method, String url, StringEntity postBody, RequestParams requestParams, AsyncHttpResponseHandler responseHandler) {
Header[] headers = new Header[]{new TokenHeader()};
Log.d("API", "Do request to: " + url);
if(method == "post") {
client.post(player.getContext(), url, headers, postBody,"application/json", responseHandler);
if(postBody != null) {
client.post(player.getContext(), url, headers, postBody,"application/json", responseHandler);
} else {
// let content type be determinded by sender
client.post(player.getContext(), url, headers, requestParams, null, responseHandler);
}
} else {
client.get(player.getContext(), url, headers, getParams, responseHandler);
client.get(player.getContext(), url, headers, requestParams, responseHandler);
}
}
@ -152,6 +163,12 @@ public class ApiRestClient {
}
}
/**
* Post string body with JWT (stringentity can be json-data)
* @param url
* @param postBody
* @param responseHandler
*/
public void post(String url, StringEntity postBody, AsyncHttpResponseHandler responseHandler) {
String jwt = player.getJWT();
if(jwt != null) {
@ -161,6 +178,21 @@ public class ApiRestClient {
}
}
/**
* Post params (ie. multipart) with JWT (stringentity can be json-data)
* @param url
* @param params
* @param responseHandler
*/
public void post(String url, RequestParams params, AsyncHttpResponseHandler responseHandler) {
String jwt = player.getJWT();
if(jwt != null) {
request("post", getAbsoluteUrl(url), null, params, responseHandler);
} else {
requestWithJWT("post", getAbsoluteUrl(url), null, params, responseHandler);
}
}
private static String getAbsoluteUrl(String relativeUrl) {
return BASE_URL + relativeUrl;
}
@ -255,6 +287,7 @@ public class ApiRestClient {
game.achievements = achievements;
gameHelper.saveAchievementsForGame(game);
}
} catch (JSONException e) {
Log.e("API","Invalid data: " + response.toString());
e.printStackTrace();
@ -274,6 +307,134 @@ public class ApiRestClient {
});
}
/**
* 1. Cut out the essential parts of the image (privacy)
* 2. send to server in one batch
* 3. if successful, remove from device to save storage.
* @param hits
*/
public void sendHitImages(final Collection<Hit> hits) {
// 1. Cut out the essential parts
GameOpenHelper gameHelper = player.getGameOpenHelper();
if(hits == null || hits.size() < 1) {
return;
}
RequestParams params = new RequestParams();
boolean noImages = true;
for(Hit hit: hits) {
Bitmap img = gameHelper.getImageForHit(hit);
if(img == null) {
Log.e("API", "no image for hit " + hit.id);
continue;
}
noImages = false;
// Crop: http://stackoverflow.com/a/31698091
int margin = (int) ((hit.points[10].x - hit.points[5].x) * 0.05);
int left = (int) (hit.points[5].x) - margin;
int right = (int) (hit.points[10].x) + margin;
int top = (int) (hit.points[6].y < hit.points[9].y ? hit.points[6].y : hit.points[9].y) - 2*margin; // a bit of forehead
int bottom = (int) (hit.points[5].y > hit.points[10].y ? hit.points[5].y : hit.points[10].y) + margin;
Rect rect = new Rect(left, top, right, bottom);
// Be sure that there is at least 1px to slice.
if(!(rect.left < rect.right && rect.top < rect.bottom)) {
Log.e("API", "Error in point positions."+" left: " + rect.left + " right: " + rect.right + " top: " + rect.top + " bottom: " + rect.bottom );
continue; // strange bug... skip it and drop the file anyway
}
// Create our resulting image (150--50),(75--25) = 200x100px
Bitmap croppedBmp = Bitmap.createBitmap(rect.right-rect.left, rect.bottom-rect.top, Bitmap.Config.ARGB_8888);
// draw source bitmap into resulting image at given position:
new Canvas(croppedBmp).drawBitmap(img, -rect.left, -rect.top, null);
ByteArrayOutputStream stream = new ByteArrayOutputStream();
croppedBmp.compress(Bitmap.CompressFormat.JPEG, 90, stream);
byte[] imageBytes = stream.toByteArray();
params.put(hit.remoteId + ":brows", new ByteArrayInputStream(imageBytes), hit.remoteId + "-brows.jpg", "image/jpeg");
Log.v("API", "add param: " + hit.remoteId + ":brows - length:" + imageBytes.length );
// NOSE
// Crop: http://stackoverflow.com/a/31698091
// reuse margin of brows
left = (int) hit.points[13].x - margin;
right = (int) hit.points[15].x + margin;
top = (int) hit.points[11].y - margin;
bottom = (int) hit.points[14].y + margin;
Rect rect2 = new Rect(left, top, right, bottom);
// Be sure that there is at least 1px to slice.
if(!(rect2.left < rect2.right && rect2.top < rect2.bottom)) {
Log.e("API", "Error in point positions."+" left: " + rect2.left + " right: " + rect2.right + " top: " + rect2.top + " bottom: " + rect2.bottom );
continue; // strange bug... skip it and drop the file anyway
}
// Create our resulting image (150--50),(75--25) = 200x100px
croppedBmp = Bitmap.createBitmap(rect2.right-rect2.left, rect2.bottom-rect2.top, Bitmap.Config.ARGB_8888);
// draw source bitmap into resulting image at given position:
new Canvas(croppedBmp).drawBitmap(img, -rect2.left, -rect2.top, null);
ByteArrayOutputStream stream2 = new ByteArrayOutputStream();
croppedBmp.compress(Bitmap.CompressFormat.JPEG, 90, stream2);
byte[] imageBytes2 = stream2.toByteArray();
params.put(hit.remoteId + ":nose", new ByteArrayInputStream(imageBytes2), hit.remoteId + "-nose.jpg", "image/jpeg");
Log.v("API", "add param: " + hit.remoteId + ":nose - length:" + imageBytes.length );
// RIGHT MOUTH CORNER
// Crop: http://stackoverflow.com/a/31698091
// reuse margin of brows to make a square image
left = (int) hit.points[24].x - margin;
right = (int) hit.points[24].x + 3 * margin;
top = (int) hit.points[24].y - 2 * margin;
bottom = (int) hit.points[24].y + 2 * margin;
Rect rect3 = new Rect(left, top, right, bottom);
// Be sure that there is at least 1px to slice.
if(!(rect3.left < rect3.right && rect3.top < rect3.bottom)) {
Log.e("API", "Error in point positions."+" left: " + rect3.left + " right: " + rect3.right + " top: " + rect3.top + " bottom: " + rect3.bottom );
continue; // strange bug... skip it and drop the file anyway
}
croppedBmp = Bitmap.createBitmap(rect3.right-rect3.left, rect3.bottom-rect3.top, Bitmap.Config.ARGB_8888);
// draw source bitmap into resulting image at given position:
new Canvas(croppedBmp).drawBitmap(img, -rect3.left, -rect3.top, null);
ByteArrayOutputStream stream3 = new ByteArrayOutputStream();
croppedBmp.compress(Bitmap.CompressFormat.JPEG, 90, stream3);
byte[] imageBytes3 = stream3.toByteArray();
params.put(hit.remoteId + ":mouth_right", new ByteArrayInputStream(imageBytes3), hit.remoteId + "-mouth_right.jpg", "image/jpeg");
Log.v("API", "add param: " + hit.remoteId + ":mouth_right - length:" + imageBytes.length );
}
// don't post if there are no images
// this happens ie. when images are all successfully send
if(noImages) {
return;
}
post("/images", params,new JsonHttpResponseHandler(){
@Override
public void onSuccess(int statusCode, Header[] headers, JSONObject response) {
// 3. remove images :-)
for(Hit hit: hits) {
player.getGameOpenHelper().clearImageForHit(hit);
}
}
@Override
public void onFailure(int statusCode, Header[] headers, String responseString, Throwable throwable) {
handleOnFailure(statusCode, headers, responseString, throwable);
}
@Override
public void onFailure(int statusCode, Header[] headers, Throwable throwable, JSONObject response) {
onFailure(statusCode, headers, response == null ? null:response.toString(), throwable);
}
});
}
public static void handleOnFailure(int statusCode, Header[] headers, String responseString, Throwable throwable) {
Log.e("API", "FAILURE ON REQUEST!");
Log.e("API", throwable.getMessage());

21
app/src/main/java/com/rubenvandeven/emotionhero/GameOpenHelper.java
View File

@ -6,6 +6,8 @@ import android.content.Intent;
import android.database.Cursor;
import android.database.sqlite.SQLiteDatabase;
import android.database.sqlite.SQLiteOpenHelper;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.graphics.PointF;
import android.util.Log;
@ -153,7 +155,7 @@ public class GameOpenHelper extends SQLiteOpenHelper {
"image BLOB," +
"PRIMARY KEY (id));" ;
private static final String[] HIT_PROJECTION = {
"id","game_id", "target_index","score","bonus","remoteId","glasses","ethnicity","age","gender","anger","contempt","disgust","fear","joy","sadness","surprise","roll","pitch","yaw","inter_ocular_distance","mouth_open","lip_press","brow_raise","nose_wrinkler","lip_depressor","brow_furrow","attention","smile","inner_brow_raiser","chin_raiser","smirk","lip_suck","upper_lip_raiser","lip_pucker","eye_closure","engagement","valence","point_0x","point_0y","point_1x","point_1y","point_2x","point_2y","point_3x","point_3y","point_4x","point_4y","point_5x","point_5y","point_6x","point_6y","point_7x","point_7y","point_8x","point_8y","point_9x","point_9y","point_10x","point_10y","point_11x","point_11y","point_12x","point_12y","point_13x","point_13y","point_14x","point_14y","point_15x","point_15y","point_16x","point_16y","point_17x","point_17y","point_18x","point_18y","point_19x","point_19y","point_20x","point_20y","point_21x","point_21y","point_22x","point_22y","point_23x","point_23y","point_24x","point_24y","point_25x","point_25y","point_26x","point_26y","point_27x","point_27y","point_28x","point_28y","point_29x","point_29y","point_30x","point_30y","point_31x","point_31y","point_32x","point_32y","point_33x","point_33y", "image"
"id","game_id", "target_index","score","bonus","remoteId","glasses","ethnicity","age","gender","anger","contempt","disgust","fear","joy","sadness","surprise","roll","pitch","yaw","inter_ocular_distance","mouth_open","lip_press","brow_raise","nose_wrinkler","lip_depressor","brow_furrow","attention","smile","inner_brow_raiser","chin_raiser","smirk","lip_suck","upper_lip_raiser","lip_pucker","eye_closure","engagement","valence","point_0x","point_0y","point_1x","point_1y","point_2x","point_2y","point_3x","point_3y","point_4x","point_4y","point_5x","point_5y","point_6x","point_6y","point_7x","point_7y","point_8x","point_8y","point_9x","point_9y","point_10x","point_10y","point_11x","point_11y","point_12x","point_12y","point_13x","point_13y","point_14x","point_14y","point_15x","point_15y","point_16x","point_16y","point_17x","point_17y","point_18x","point_18y","point_19x","point_19y","point_20x","point_20y","point_21x","point_21y","point_22x","point_22y","point_23x","point_23y","point_24x","point_24y","point_25x","point_25y","point_26x","point_26y","point_27x","point_27y","point_28x","point_28y","point_29x","point_29y","point_30x","point_30y","point_31x","point_31y","point_32x","point_32y","point_33x","point_33y"
};
private static final String GAME_ACH_TABLE_NAME = "game_achievements";
@ -207,6 +209,23 @@ public class GameOpenHelper extends SQLiteOpenHelper {
}
public Bitmap getImageForHit(Hit hit) {
String[] params = { Long.toString(hit.id) };
Cursor c = getReadableDatabase().rawQuery("SELECT image FROM hits h WHERE h.id = ?", params);
c.moveToFirst();
byte[] imageBytes= c.getBlob(0);
if(imageBytes == null)
{
return null;
}
return BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.length);
}
public void clearImageForHit(Hit hit) {
String[] params = { Long.toString(hit.id) };
getWritableDatabase().rawQuery("UPDATE hits SET image = NULL WHERE id = ?", params);
}
public void insertGame(Game game) {
// Gets the data repository in write mode
SQLiteDatabase db = getWritableDatabase();

3
app/src/main/java/com/rubenvandeven/emotionhero/GamingActivity.java
View File

@ -65,6 +65,7 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL
public HashMap<Integer, Integer> soundIds = new HashMap<>();
final static int SOUND_SCORE = 1;
final static int SOUND_BONUS = 2;
protected Player player;
@ -154,6 +155,7 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL
createSoundPool(); // instantiate SoundPool in sound
soundIds.put(SOUND_SCORE, sound.load(this, R.raw.score2, 1));
soundIds.put(SOUND_BONUS, sound.load(this, R.raw.scorebonus, 1));
// StoryDialogFragment storyDialog = new StoryDialogFragment();
// storyDialog.show(getSupportFragmentManager(), "StoryDialog");
@ -345,6 +347,7 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL
.setContentType(AudioAttributes.CONTENT_TYPE_SONIFICATION)
.build();
sound = new SoundPool.Builder()
.setMaxStreams(6) // allow sounds to ring while another is started
.setAudioAttributes(attributes)
.build();
}

1
app/src/main/java/com/rubenvandeven/emotionhero/HighscoreActivity.java
View File

@ -291,6 +291,7 @@ public class HighscoreActivity extends AppCompatActivity {
startLvlButton.setVisibility(View.INVISIBLE);
}
nextLvlButton.setVisibility(View.GONE);
if(scenario.isFinalLevel()) {
nextLvlButton.setVisibility(View.INVISIBLE);
} else {

18
app/src/main/java/com/rubenvandeven/emotionhero/MirrorMenuActivity.java
View File

@ -165,14 +165,14 @@ public class MirrorMenuActivity extends AppCompatActivity implements Detector.Im
startButton.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
Intent intent;
if(!player.getPlayerInfo().canContinueLevel()) {
intent = new Intent(MirrorMenuActivity.this, IntroductionActivity.class);
} else {
loadGameDialog = ProgressDialog.show(MirrorMenuActivity.this, "",
MirrorMenuActivity.this.getResources().getString(R.string.load_game_activity), true);
intent = new Intent(MirrorMenuActivity.this, GamingActivity.class);
}
Intent intent = new Intent(MirrorMenuActivity.this, IntroductionActivity.class);
// if(!player.getPlayerInfo().canContinueLevel()) {
// intent = new Intent(MirrorMenuActivity.this, IntroductionActivity.class);
// } else {
// loadGameDialog = ProgressDialog.show(MirrorMenuActivity.this, "",
// MirrorMenuActivity.this.getResources().getString(R.string.load_game_activity), true);
// intent = new Intent(MirrorMenuActivity.this, GamingActivity.class);
// }
startActivity(intent);
detector.stop();
detector = null;
@ -181,6 +181,8 @@ public class MirrorMenuActivity extends AppCompatActivity implements Detector.Im
});
// merge with 'start button'
if(!player.getPlayerInfo().canContinueLevel()) {
highscoresButton.setVisibility(View.GONE);
} else {

3
app/src/main/java/com/rubenvandeven/emotionhero/ProgressActivity.java
View File

@ -1,5 +1,6 @@
package com.rubenvandeven.emotionhero;
import android.app.ProgressDialog;
import android.content.Intent;
import android.graphics.Typeface;
import android.support.v7.app.AppCompatActivity;
@ -130,6 +131,8 @@ public class ProgressActivity extends AppCompatActivity {
intent.putExtra(HighscoreActivity.INTENT_EXTRA_LVL_ID, scenario.id);
} else {
// "PLAY NOW!"
ProgressDialog dialog = ProgressDialog.show(getApplicationContext(), "",
getApplicationContext().getResources().getString(R.string.load_game_activity), true);
intent = new Intent(ProgressActivity.this, GamingActivity.class);
intent.putExtra(GamingActivity.INTENT_EXTRA_SCENARIO, scenario.id);
}

2
app/src/main/java/com/rubenvandeven/emotionhero/ReviewActivity.java
View File

@ -103,6 +103,8 @@ public class ReviewActivity extends AppCompatActivity {
scoreProgressBar.setVisibility(View.GONE);
}
});
// try submitting images if not yet done.
player.api.sendHitImages(game.hits.values());
}
}
};

29
app/src/main/java/com/rubenvandeven/emotionhero/Scenario.java
View File

@ -3,6 +3,7 @@ package com.rubenvandeven.emotionhero;
import android.graphics.Bitmap;
import android.graphics.Canvas;
import android.graphics.Color;
import android.graphics.Matrix;
import android.graphics.Paint;
import android.graphics.RectF;
import android.support.v8.renderscript.RenderScript;
@ -172,8 +173,14 @@ public class Scenario {
Hit hit = new Hit(target, game, currentFace);
_activity.sound.play(_activity.soundIds.get(_activity.SOUND_SCORE), 1, 1,
1,0, hit.score / 200f + 0.5f ); // play back the sound slower
2,0, hit.score / 200f + 0.5f ); // play back the sound slower
// depending on hit value
if(hit.bonus > 0) {
_activity.sound.play(_activity.soundIds.get(_activity.SOUND_BONUS), 0.7f, 0.7f,
1,0, hit.bonus / 100f + 0.8f ); // play back the sound slower
// depending on bonus value
}
target.isHit = true;
@ -183,13 +190,25 @@ public class Scenario {
// convert NV21 byteArrayFrame from camera to RGB bitmap.
Frame.ByteArrayFrame byteArrayFrame = (Frame.ByteArrayFrame) currentFrame;
outputBitmap = Nv21Image.nv21ToBitmap(rs, byteArrayFrame.getByteArray(), byteArrayFrame.getWidth(), byteArrayFrame.getHeight());
Frame.ROTATE rotation = byteArrayFrame.getTargetRotation();
// ie BY_90_CCW -90.0
Log.v("Scenario", "frame rotation: " + rotation.toString() + " " + rotation.toDouble());
// process rotation... (maybe faster if done on byte array?)
if(rotation != Frame.ROTATE.NO_ROTATION) {
Matrix matrix = new Matrix();
matrix.postRotate((int) rotation.toDouble());
// int width = rotation == Frame.ROTATE.BY_180 ? outputBitmap.getWidth() : outputBitmap.getHeight();
// int height = rotation == Frame.ROTATE.BY_180 ? outputBitmap.getHeight() : outputBitmap.getWidth();
outputBitmap = Bitmap.createBitmap(outputBitmap , 0, 0, outputBitmap .getWidth(), outputBitmap .getHeight(), matrix, true);
}
Canvas canvas = new Canvas(outputBitmap);
RectF rect = Hit.getBoundingboxForPoints(currentFace.getFacePoints());
canvas.drawRect(rect, squarePaint);
// Deprecated: now we send only fragments of the images anyway.
// Canvas canvas = new Canvas(outputBitmap);
// RectF rect = Hit.getBoundingboxForPoints(currentFace.getFacePoints());
// canvas.drawRect(rect, squarePaint);
/*
Also possible: only grayscale image - only loops over first part of the bytearray
Also possible: only grayscale image - only loops over first part of the bytearray, is it faster?
int[] pixels;
int p;
int size = width*height;

2
app/src/main/res/layout/activity_mirror_menu.xml
View File

@ -40,7 +40,7 @@
android:textColor="@color/textPrimary"
android:textAllCaps="true"
android:layout_marginBottom="@dimen/fab_margin"
android:text="@string/highscores"
android:text="@string/continueBtn"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:id="@+id/highscoresButton"