r
/
emotionhero
1
0
Fork 0
Code Issues 1 Pull Requests Releases Wiki Activity

Add sound and proper tick for scenario

This commit is contained in:
Ruben 2016-08-19 12:38:59 +01:00
parent 1dcb08a932
commit 1503b65439
5 changed files with 261 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
app/src/main/java/com/rubenvandeven/emotionhero/GamingActivity.java
View File

@ -2,9 +2,12 @@ package com.rubenvandeven.emotionhero;
import android.Manifest;
import android.annotation.SuppressLint;
import android.annotation.TargetApi;
import android.content.pm.PackageManager;
import android.graphics.Bitmap;
import android.graphics.Canvas;
import android.media.AudioAttributes;
import android.media.AudioManager;
import android.media.SoundPool;
import android.os.Build;
import android.support.v4.app.ActivityCompat;
import android.support.v7.app.ActionBar;
import android.support.v7.app.AppCompatActivity;
@ -25,6 +28,7 @@ import com.affectiva.android.affdex.sdk.detector.CameraDetector;
import com.affectiva.android.affdex.sdk.detector.Detector;
import com.affectiva.android.affdex.sdk.detector.Face;
import java.util.HashMap;
import java.util.List;
/**
@ -123,6 +127,12 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL
TextView paramText;
public SoundPool sound;
public HashMap<Integer, Integer> soundIds = new HashMap<>();
final static int SOUND_SCORE = 1;
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
@ -182,6 +192,7 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL
}
}
setMeasuredDimension(width,height);
// setMeasuredDimension(1,1); // this DOES increase performance....
}
};
RelativeLayout.LayoutParams params = new RelativeLayout.LayoutParams(ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.WRAP_CONTENT);
@ -191,16 +202,15 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL
videoLayout.addView(cameraPreview,0);
currentScenario = new ScenarioAnger();
currentScenario = new ScenarioAnger(this);
scenarioView = new ScenarioView(this, currentScenario);
RelativeLayout.LayoutParams scenarioViewParams = new RelativeLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT);
videoLayout.addView(scenarioView, 1, scenarioViewParams);
// new LayoutParams(LayoutParams.FILL_PARENT, LayoutParams.WRAP_CONTENT)
// videoLayout.addView(scenarioView, 200, 100);
createSoundPool(); // instantiate SoundPool in sound
soundIds.put(SOUND_SCORE, sound.load(this, R.raw.score2, 1));
// startDetector();
}
@Override
@ -254,14 +264,16 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL
if(detector == null)
{
// SurfaceView surfaceView = (SurfaceView) findViewById(R.id.surfaceView);
detector = new CameraDetector(this, CameraDetector.CameraType.CAMERA_FRONT, cameraPreview);
detector = new CameraDetector(this, CameraDetector.CameraType.CAMERA_FRONT, cameraPreview, 1, Detector.FaceDetectorMode.LARGE_FACES);
detector.setLicensePath("emotionhero_dev.license");
detector.setMaxProcessRate(10);
detector.setDetectAllEmotions(true);
detector.setDetectAllAppearances(false);
detector.setDetectAllEmojis(false);
detector.setDetectAllExpressions(false);
detector.setMaxProcessRate(12);
detector.setMaxProcessRate(20);
detector.setImageListener(this);
detector.setOnCameraEventListener(this);
@ -269,7 +281,7 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL
}
detector.start();
mContentView.setText("STARTING...");
setText("STARTING...");
Log.d(LOG_TAG, Boolean.toString(detector.isRunning()));
}
}
@ -321,10 +333,12 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL
* Detector callback gives the faces found so we can match their scores to the given scenario.
*/
public void onImageResults(List<Face> list, Frame frame, float timestamp) {
// frame.getOriginalBitmapFrame()
// Log.e(LOG_TAG, "RESULT! faces: " + Integer.toString(list.size()) + " t: " + Float.toString(timestamp) + "s" );
if(!currentScenario.isWithinTime(timestamp))
// if(!currentScenario.isWithinTime(timestamp))
if(currentScenario.isFinished())
{
mContentView.setText(String.format("LEVEL ENDED\nScore: %.2f", currentScenario.getTotalScore()));
setText(String.format("LEVEL ENDED\nScore: %.2f", currentScenario.getTotalScore()));
stopDetector();
restartButton.setVisibility(View.VISIBLE);
return;
@ -332,12 +346,12 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL
if (list == null)
return;
if (list.size() == 0) {
mContentView.setText("NO FACE FOUND");
// mContentView.setText("NO FACE FOUND"); // this happens in onFaceDetectionStopped
} else {
hideText(); // hide textView as we want as few elements as possible.
Face face = list.get(0);
currentScenario.validateFaceOnTime(face, timestamp);
currentScenario.setCurrentAttributeScoresForFace(face);
scenarioView.setCurrentAttributeScoresForFace(face);
mContentView.setText(String.format("SCORE \n%.2f",currentScenario.getTotalScore()));
// String paramString = "";
// paramString += "Anger " + String.format("%02.2f", face.emotions.getAnger()) + "%\n";
@ -394,14 +408,53 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL
@Override
public void onFaceDetectionStarted()
{
mContentView.setText("START!");
setText("START!");
currentScenario.start();
}
@Override
public void onFaceDetectionStopped()
{
mContentView.setText("No face found...");
// paramText.setText("No face found");
setText("No face found...");
currentScenario.pause();
}
public void setText(String text)
{
mContentView.setVisibility(View.VISIBLE);
mContentView.setText(text);
}
public void hideText()
{
mContentView.setVisibility(View.GONE);
}
// http://stackoverflow.com/a/27552576
protected void createSoundPool() {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
createNewSoundPool();
} else {
createOldSoundPool();
}
}
// http://stackoverflow.com/a/27552576
@TargetApi(Build.VERSION_CODES.LOLLIPOP)
protected void createNewSoundPool(){
AudioAttributes attributes = new AudioAttributes.Builder()
.setUsage(AudioAttributes.USAGE_GAME)
.setContentType(AudioAttributes.CONTENT_TYPE_SONIFICATION)
.build();
sound = new SoundPool.Builder()
.setAudioAttributes(attributes)
.build();
}
// http://stackoverflow.com/a/27552576
@SuppressWarnings("deprecation")
protected void createOldSoundPool(){
sound = new SoundPool(5, AudioManager.STREAM_MUSIC,0);
}
}

147
app/src/main/java/com/rubenvandeven/emotionhero/Scenario.java
View File

@ -1,16 +1,12 @@
package com.rubenvandeven.emotionhero;
import android.graphics.Bitmap;
import android.graphics.Canvas;
import android.graphics.Color;
import android.graphics.Paint;
import android.util.Log;
import com.affectiva.android.affdex.sdk.detector.Face;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
/**
* Created by ruben on 16/08/16.
@ -18,10 +14,36 @@ import java.util.Map;
abstract public class Scenario {
static int DESIRED_FPS = 25;
float duration = 0;
/**
* @deprecated
*/
long startTime = 0;
/**
* The timer that provides the tick
*/
Timer timer;
/**
* Increment on each tick
*/
float runningTime = -1;
boolean isRunning = false;
private GamingActivity _activity;
/**
* The scorres in this moment, as to draw them on the screen.
* Indexes are Emotion ordinals
*/
private Map<Emotion, Float> currentAttributeScores = new HashMap<>();
ArrayList<Target> targets = new ArrayList<>();
abstract void createScenario();
@ -30,6 +52,7 @@ abstract public class Scenario {
public Emotion emotion;
public float value;
public float timestamp;
public Score score;
}
ArrayList<Score> scores = new ArrayList<>();
@ -40,20 +63,68 @@ abstract public class Scenario {
* Extra bonus given
*/
public boolean isSpotOn = false;
/**
* The target the score is awarded for
*/
public Target target;
}
/**
* Constructor
*/
public Scenario()
public Scenario(GamingActivity activity)
{
createScenario();
timer = new Timer("ScenarioTimer");
TimerTask tickTask;
tickTask = new TimerTask() {
@Override
public void run() {
// if (System.currentTimeMillis() - scheduledExecutionTime() >=
// MAX_TARDINESS)
// return; // Too late; skip this execution.
tick();
}
};
timer.schedule(tickTask, 0, 1000/DESIRED_FPS);
_activity = activity;
}
/**
* To be called on each timer tick
*/
public void tick()
{
if(!isRunning)
return;
runningTime += 1.0f/DESIRED_FPS;
if(isFinished()) {
stop();
return;
}
for (int i = targets.size() - 1; i >= 0; i--) {
Target target = targets.get(i);
// skip targets that are already scored
if(target.score != null) {
continue;
}
if(target.timestamp <= runningTime) {
float scored_value = currentAttributeScores.get(target.emotion);
float required_value = target.value;
Score score = new Score();
score.value = Math.round(100 - Math.abs(scored_value-required_value));
scores.add(score);
//
_activity.sound.play(_activity.soundIds.get(_activity.SOUND_SCORE), 1, 1,
1,0, score.value / 200f + 0.5f ); // play back the sound slower
// depending on score value
target.score = score;
}
}
}
/**
* Add a target on given timestamp
* @param emotion
@ -63,9 +134,9 @@ abstract public class Scenario {
public void setTarget(Emotion emotion, float value, float timestamp)
{
// Log.e(GamingActivity.LOG_TAG, "Set target:" + Float.toString(timestamp) + " " + Float.toString(duration));
if(timestamp > duration)
if((timestamp + 1) > duration)
{
duration = timestamp;
duration = timestamp + 1; // always a bit onger than last target, as it otherwise the game does not finish pretty
}
Target target = new Target();
target.timestamp = timestamp;
@ -91,6 +162,11 @@ abstract public class Scenario {
setTarget(emotion, value, timestamp);
}
/**
* @deprecated use @see tick()
* @param face
* @param timestamp
*/
public void validateFaceOnTime(Face face, float timestamp)
{
// TODO: interpolation of time
@ -101,7 +177,7 @@ abstract public class Scenario {
float required_value = target.value;
Score score = new Score();
score.value = 100 - Math.abs(scored_value-required_value);
score.target = target;
target.score = score;
scores.add(score);
}
}
@ -135,21 +211,46 @@ abstract public class Scenario {
* Get the time within the scenario (so since start() has been called)
*/
public float getTime() {
return runningTime;
// if not started, don't move the labels, if started, move them by diff_y
if(startTime == 0) {
return 0;
} else {
float diff_t = ((System.currentTimeMillis() - startTime)) / (float) 1000;
if(diff_t > duration) { // never larger than scenario duration
return duration;
}
return diff_t;
}
// if(startTime == 0) {
// return 0;
// } else {
// float diff_t = ((System.currentTimeMillis() - startTime)) / (float) 1000;
// if(diff_t > duration) { // never larger than scenario duration
// return duration;
// }
// return diff_t;
// }
}
public void start()
{
startTime = System.currentTimeMillis();
isRunning = true;
}
public void pause() {
isRunning = false;
}
public void stop()
{
isRunning = false;
}
// TODO: create AttributeScoreCollection class, with this method.
public void setCurrentAttributeScoresForFace(Face face)
{
for(Emotion emotion: Emotion.values()) {
currentAttributeScores.put(emotion, emotion.getValueFromFace(face));
}
}
public boolean isFinished()
{
return runningTime > duration;
}
// TODO: create a 'tick' that checks all current values with requirements and increases the timer etc

5
app/src/main/java/com/rubenvandeven/emotionhero/ScenarioAnger.java
View File

@ -7,6 +7,11 @@ import android.util.Log;
*/
public class ScenarioAnger extends Scenario {
public ScenarioAnger(GamingActivity activity) {
super(activity);
}
void createScenario()
{
Log.d(GamingActivity.LOG_TAG, "CREATE SCENARIO: anger");

87
app/src/main/java/com/rubenvandeven/emotionhero/ScenarioView.java
View File

@ -6,8 +6,7 @@ import android.graphics.Color;
import android.graphics.Paint;
import android.graphics.Path;
import android.graphics.PixelFormat;
import android.util.AttributeSet;
import android.util.Log;
import android.graphics.Typeface;
import android.view.SurfaceHolder;
import android.view.SurfaceView;
@ -32,6 +31,14 @@ public class ScenarioView extends SurfaceView implements SurfaceHolder.Callback
*/
private Map<Emotion, Float> currentAttributeScores = new HashMap<>();
//avoid object instantiation on each onDraw
private Map<Emotion, Paint> emoPaints = new HashMap<>();
private Map<Emotion, Paint> emoOutlinePaints = new HashMap<>();
private Map<Emotion, Paint> emoScoredPaints = new HashMap<>();
private Paint mainPaint = new Paint();
private Paint attrScorePaint = new Paint();
private Paint linePaint = new Paint();
// see: http://blog.danielnadeau.io/2012/01/android-canvas-beginners-tutorial.html
class PanelThread extends Thread {
@ -58,6 +65,7 @@ public class ScenarioView extends SurfaceView implements SurfaceHolder.Callback
c = null; //set to false and loop ends, stopping thread
try {
// c = _surfaceHolder.lockCanvas(null);
c = _surfaceHolder.lockCanvas(null);
synchronized (_surfaceHolder) {
//Insert methods to modify positions of items in onDraw()
@ -76,6 +84,43 @@ public class ScenarioView extends SurfaceView implements SurfaceHolder.Callback
super(context);
getHolder().addCallback(this);
_scenario = s;
//setup paints for drawing
mainPaint.setColor(Color.GRAY);
mainPaint.setTextSize(40);
mainPaint.setTypeface(Typeface.SANS_SERIF);
linePaint.setColor(Color.GRAY);
linePaint.setStrokeWidth(5);
attrScorePaint.setColor(Color.DKGRAY);
for(Emotion emotion: Emotion.values()) {
Paint emoPaint = new Paint();
emoPaint.setTextSize(20);
emoPaint.setColor(emotion.getColor());
emoPaints.put(emotion, emoPaint);
Paint emoPaintOutline = new Paint();
emoPaintOutline.setColor(emotion.getColor());
emoPaintOutline.setStyle(Paint.Style.STROKE);
emoPaintOutline.setStrokeWidth(2);
emoOutlinePaints.put(emotion, emoPaintOutline);
Paint emoScoredPaint = new Paint();
float darkenFactor = 0.4f;
int red = (int) ((Color.red(emotion.getColor()) * darkenFactor));
int green = (int) ((Color.green(emotion.getColor()) * darkenFactor));
int blue = (int) ((Color.blue(emotion.getColor()) * darkenFactor));
int emoLightenedColor = Color.rgb(red, green, blue);
emoScoredPaint.setColor(emoLightenedColor);
emoScoredPaint.setTextSize(20);
emoScoredPaint.setStrokeWidth(2);
emoScoredPaint.setStyle(Paint.Style.FILL_AND_STROKE);
emoScoredPaints.put(emotion, emoScoredPaint);
}
}
@ -96,15 +141,6 @@ public class ScenarioView extends SurfaceView implements SurfaceHolder.Callback
float step_y = (canvas.getWidth() * used_width) / Emotion.values().length;
float max_ball_radius = step_y * (float) 0.8 / 2;
// bottom at 80%;
Paint mainPaint = new Paint();
mainPaint.setColor(Color.GRAY);
Paint linePaint = new Paint();
linePaint.setColor(Color.GRAY);
linePaint.setStrokeWidth(5);
// canvas.drawLine(0, bottomline_height, width, bottomline_height, linePaint);
@ -117,22 +153,16 @@ public class ScenarioView extends SurfaceView implements SurfaceHolder.Callback
if(value < 5) {
value = 5;
}
Paint emoPaint = new Paint();
Paint emoPaintOutline = new Paint();
emoPaint.setTextSize(20);
emoPaint.setColor(emotion.getColor());
emoPaintOutline.setColor(emotion.getColor());
emoPaintOutline.setStyle(Paint.Style.STROKE);
emoPaintOutline.setStrokeWidth(2);
float cx = padding_left + (step_y * emotion.ordinal() + step_y / 2);
float cy = bottomline_height;
canvas.drawCircle(cx, cy, max_ball_radius, mainPaint);
canvas.drawCircle(cx, cy, max_ball_radius, emoPaintOutline);
canvas.drawCircle(cx, cy, max_ball_radius, emoOutlinePaints.get(emotion));
canvas.drawCircle(cx, cy, max_ball_radius * value/100, emoPaint);
// canvas.drawCircle(cx, cy, max_ball_radius * value/100, emoPaints.get(emotion.ordinal()));
canvas.drawCircle(cx, cy, max_ball_radius * value/100, attrScorePaint);
Path emoNamePath = new Path();
emoNamePath.moveTo(cx, cy + max_ball_radius * 1.5f);
@ -141,7 +171,7 @@ public class ScenarioView extends SurfaceView implements SurfaceHolder.Callback
emoNamePath.rLineTo(1000,1000);
// canvas.drawText(emotion.toString(), cx, cy + max_ball_radius * (float) 1.3, emoPaint);
canvas.drawTextOnPath(emotion.toString(), emoNamePath, 0, 0, emoPaint);
canvas.drawTextOnPath(emotion.toString(), emoNamePath, 0, 0, emoPaints.get(emotion));
}
// Draw targets:
@ -150,21 +180,28 @@ public class ScenarioView extends SurfaceView implements SurfaceHolder.Callback
float diff_y = sec_height * _scenario.getTime();
for(Scenario.Target target: _scenario.getTargets()) {
Paint emoPaint = new Paint();
emoPaint.setColor(target.emotion.getColor());
Paint targetPaint = (target.score == null) ? emoPaints.get(target.emotion) : emoScoredPaints.get(target.emotion);
float cy = bottomline_height - (target.timestamp * sec_height) + diff_y;
float cx = padding_left + (step_y * target.emotion.ordinal() + step_y / 2);
canvas.drawCircle(cx, cy, max_ball_radius * target.value/100, emoPaint);
canvas.drawCircle(cx, cy, max_ball_radius * target.value/100, targetPaint);
if(target.score != null) {
canvas.drawText(Float.toString(target.score.value), cx, cy, targetPaint);
}
// String target_text = target.emotion.toString() + " " + target.value + "%";
// canvas.drawText(target_text, cx, y_pos + diff_y , emoPaint);
}
canvas.drawText("Total: " + Float.toString(_scenario.getTotalScore()), 50, 50, mainPaint);
}
// TODO: create AttributeScoreCollection class, with this method.
public void setCurrentAttributeScoresForFace(Face face)
{
for(Emotion emotion: Emotion.values()) {

BIN
app/src/main/res/raw/score2.mp3 Normal file
View File

Binary file not shown.