diff --git a/app/src/main/java/com/rubenvandeven/emotionhero/GamingActivity.java b/app/src/main/java/com/rubenvandeven/emotionhero/GamingActivity.java index cea872f..b29eaf2 100644 --- a/app/src/main/java/com/rubenvandeven/emotionhero/GamingActivity.java +++ b/app/src/main/java/com/rubenvandeven/emotionhero/GamingActivity.java @@ -2,9 +2,12 @@ package com.rubenvandeven.emotionhero; import android.Manifest; import android.annotation.SuppressLint; +import android.annotation.TargetApi; import android.content.pm.PackageManager; -import android.graphics.Bitmap; -import android.graphics.Canvas; +import android.media.AudioAttributes; +import android.media.AudioManager; +import android.media.SoundPool; +import android.os.Build; import android.support.v4.app.ActivityCompat; import android.support.v7.app.ActionBar; import android.support.v7.app.AppCompatActivity; @@ -25,6 +28,7 @@ import com.affectiva.android.affdex.sdk.detector.CameraDetector; import com.affectiva.android.affdex.sdk.detector.Detector; import com.affectiva.android.affdex.sdk.detector.Face; +import java.util.HashMap; import java.util.List; /** @@ -123,6 +127,12 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL TextView paramText; + public SoundPool sound; + public HashMap soundIds = new HashMap<>(); + + final static int SOUND_SCORE = 1; + + @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); @@ -182,6 +192,7 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL } } setMeasuredDimension(width,height); +// setMeasuredDimension(1,1); // this DOES increase performance.... } }; RelativeLayout.LayoutParams params = new RelativeLayout.LayoutParams(ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.WRAP_CONTENT); @@ -191,16 +202,15 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL videoLayout.addView(cameraPreview,0); - currentScenario = new ScenarioAnger(); + currentScenario = new ScenarioAnger(this); scenarioView = new ScenarioView(this, currentScenario); RelativeLayout.LayoutParams scenarioViewParams = new RelativeLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT); videoLayout.addView(scenarioView, 1, scenarioViewParams); -// new LayoutParams(LayoutParams.FILL_PARENT, LayoutParams.WRAP_CONTENT) -// videoLayout.addView(scenarioView, 200, 100); + createSoundPool(); // instantiate SoundPool in sound + soundIds.put(SOUND_SCORE, sound.load(this, R.raw.score2, 1)); -// startDetector(); } @Override @@ -254,14 +264,16 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL if(detector == null) { // SurfaceView surfaceView = (SurfaceView) findViewById(R.id.surfaceView); - detector = new CameraDetector(this, CameraDetector.CameraType.CAMERA_FRONT, cameraPreview); + + detector = new CameraDetector(this, CameraDetector.CameraType.CAMERA_FRONT, cameraPreview, 1, Detector.FaceDetectorMode.LARGE_FACES); detector.setLicensePath("emotionhero_dev.license"); + detector.setMaxProcessRate(10); detector.setDetectAllEmotions(true); detector.setDetectAllAppearances(false); detector.setDetectAllEmojis(false); detector.setDetectAllExpressions(false); - detector.setMaxProcessRate(12); + detector.setMaxProcessRate(20); detector.setImageListener(this); detector.setOnCameraEventListener(this); @@ -269,7 +281,7 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL } detector.start(); - mContentView.setText("STARTING..."); + setText("STARTING..."); Log.d(LOG_TAG, Boolean.toString(detector.isRunning())); } } @@ -321,10 +333,12 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL * Detector callback gives the faces found so we can match their scores to the given scenario. */ public void onImageResults(List list, Frame frame, float timestamp) { +// frame.getOriginalBitmapFrame() // Log.e(LOG_TAG, "RESULT! faces: " + Integer.toString(list.size()) + " t: " + Float.toString(timestamp) + "s" ); - if(!currentScenario.isWithinTime(timestamp)) +// if(!currentScenario.isWithinTime(timestamp)) + if(currentScenario.isFinished()) { - mContentView.setText(String.format("LEVEL ENDED\nScore: %.2f", currentScenario.getTotalScore())); + setText(String.format("LEVEL ENDED\nScore: %.2f", currentScenario.getTotalScore())); stopDetector(); restartButton.setVisibility(View.VISIBLE); return; @@ -332,12 +346,12 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL if (list == null) return; if (list.size() == 0) { - mContentView.setText("NO FACE FOUND"); +// mContentView.setText("NO FACE FOUND"); // this happens in onFaceDetectionStopped } else { + hideText(); // hide textView as we want as few elements as possible. Face face = list.get(0); - currentScenario.validateFaceOnTime(face, timestamp); + currentScenario.setCurrentAttributeScoresForFace(face); scenarioView.setCurrentAttributeScoresForFace(face); - mContentView.setText(String.format("SCORE \n%.2f",currentScenario.getTotalScore())); // String paramString = ""; // paramString += "Anger " + String.format("%02.2f", face.emotions.getAnger()) + "%\n"; @@ -394,14 +408,53 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL @Override public void onFaceDetectionStarted() { - mContentView.setText("START!"); + setText("START!"); currentScenario.start(); } @Override public void onFaceDetectionStopped() { - mContentView.setText("No face found..."); -// paramText.setText("No face found"); + setText("No face found..."); + currentScenario.pause(); + } + + public void setText(String text) + { + mContentView.setVisibility(View.VISIBLE); + mContentView.setText(text); + } + + public void hideText() + { + mContentView.setVisibility(View.GONE); + } + + +// http://stackoverflow.com/a/27552576 + protected void createSoundPool() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + createNewSoundPool(); + } else { + createOldSoundPool(); + } + } + +// http://stackoverflow.com/a/27552576 + @TargetApi(Build.VERSION_CODES.LOLLIPOP) + protected void createNewSoundPool(){ + AudioAttributes attributes = new AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_GAME) + .setContentType(AudioAttributes.CONTENT_TYPE_SONIFICATION) + .build(); + sound = new SoundPool.Builder() + .setAudioAttributes(attributes) + .build(); + } + +// http://stackoverflow.com/a/27552576 + @SuppressWarnings("deprecation") + protected void createOldSoundPool(){ + sound = new SoundPool(5, AudioManager.STREAM_MUSIC,0); } } diff --git a/app/src/main/java/com/rubenvandeven/emotionhero/Scenario.java b/app/src/main/java/com/rubenvandeven/emotionhero/Scenario.java index a345fd4..2b40f57 100644 --- a/app/src/main/java/com/rubenvandeven/emotionhero/Scenario.java +++ b/app/src/main/java/com/rubenvandeven/emotionhero/Scenario.java @@ -1,16 +1,12 @@ package com.rubenvandeven.emotionhero; -import android.graphics.Bitmap; -import android.graphics.Canvas; -import android.graphics.Color; -import android.graphics.Paint; -import android.util.Log; - import com.affectiva.android.affdex.sdk.detector.Face; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; +import java.util.Timer; +import java.util.TimerTask; /** * Created by ruben on 16/08/16. @@ -18,10 +14,36 @@ import java.util.Map; abstract public class Scenario { + static int DESIRED_FPS = 25; + float duration = 0; + /** + * @deprecated + */ long startTime = 0; + /** + * The timer that provides the tick + */ + Timer timer; + + /** + * Increment on each tick + */ + float runningTime = -1; + + + boolean isRunning = false; + + private GamingActivity _activity; + + /** + * The scorres in this moment, as to draw them on the screen. + * Indexes are Emotion ordinals + */ + private Map currentAttributeScores = new HashMap<>(); + ArrayList targets = new ArrayList<>(); abstract void createScenario(); @@ -30,6 +52,7 @@ abstract public class Scenario { public Emotion emotion; public float value; public float timestamp; + public Score score; } ArrayList scores = new ArrayList<>(); @@ -40,20 +63,68 @@ abstract public class Scenario { * Extra bonus given */ public boolean isSpotOn = false; - /** - * The target the score is awarded for - */ - public Target target; } /** * Constructor */ - public Scenario() + public Scenario(GamingActivity activity) { createScenario(); + timer = new Timer("ScenarioTimer"); + TimerTask tickTask; + tickTask = new TimerTask() { + @Override + public void run() { + // if (System.currentTimeMillis() - scheduledExecutionTime() >= + // MAX_TARDINESS) + // return; // Too late; skip this execution. + tick(); + } + }; + timer.schedule(tickTask, 0, 1000/DESIRED_FPS); + _activity = activity; } + /** + * To be called on each timer tick + */ + public void tick() + { + if(!isRunning) + return; + + runningTime += 1.0f/DESIRED_FPS; + + if(isFinished()) { + stop(); + return; + } + + for (int i = targets.size() - 1; i >= 0; i--) { + Target target = targets.get(i); + // skip targets that are already scored + if(target.score != null) { + continue; + } + if(target.timestamp <= runningTime) { + float scored_value = currentAttributeScores.get(target.emotion); + float required_value = target.value; + Score score = new Score(); + score.value = Math.round(100 - Math.abs(scored_value-required_value)); + scores.add(score); + + // + _activity.sound.play(_activity.soundIds.get(_activity.SOUND_SCORE), 1, 1, + 1,0, score.value / 200f + 0.5f ); // play back the sound slower + // depending on score value + target.score = score; + } + } + + } + + /** * Add a target on given timestamp * @param emotion @@ -63,9 +134,9 @@ abstract public class Scenario { public void setTarget(Emotion emotion, float value, float timestamp) { // Log.e(GamingActivity.LOG_TAG, "Set target:" + Float.toString(timestamp) + " " + Float.toString(duration)); - if(timestamp > duration) + if((timestamp + 1) > duration) { - duration = timestamp; + duration = timestamp + 1; // always a bit onger than last target, as it otherwise the game does not finish pretty } Target target = new Target(); target.timestamp = timestamp; @@ -91,6 +162,11 @@ abstract public class Scenario { setTarget(emotion, value, timestamp); } + /** + * @deprecated use @see tick() + * @param face + * @param timestamp + */ public void validateFaceOnTime(Face face, float timestamp) { // TODO: interpolation of time @@ -101,7 +177,7 @@ abstract public class Scenario { float required_value = target.value; Score score = new Score(); score.value = 100 - Math.abs(scored_value-required_value); - score.target = target; + target.score = score; scores.add(score); } } @@ -135,21 +211,46 @@ abstract public class Scenario { * Get the time within the scenario (so since start() has been called) */ public float getTime() { + return runningTime; // if not started, don't move the labels, if started, move them by diff_y - if(startTime == 0) { - return 0; - } else { - float diff_t = ((System.currentTimeMillis() - startTime)) / (float) 1000; - if(diff_t > duration) { // never larger than scenario duration - return duration; - } - return diff_t; - } +// if(startTime == 0) { +// return 0; +// } else { +// float diff_t = ((System.currentTimeMillis() - startTime)) / (float) 1000; +// if(diff_t > duration) { // never larger than scenario duration +// return duration; +// } +// return diff_t; +// } } public void start() { startTime = System.currentTimeMillis(); + isRunning = true; + } + + public void pause() { + isRunning = false; + } + + public void stop() + { + isRunning = false; + } + + + // TODO: create AttributeScoreCollection class, with this method. + public void setCurrentAttributeScoresForFace(Face face) + { + for(Emotion emotion: Emotion.values()) { + currentAttributeScores.put(emotion, emotion.getValueFromFace(face)); + } + } + + public boolean isFinished() + { + return runningTime > duration; } // TODO: create a 'tick' that checks all current values with requirements and increases the timer etc diff --git a/app/src/main/java/com/rubenvandeven/emotionhero/ScenarioAnger.java b/app/src/main/java/com/rubenvandeven/emotionhero/ScenarioAnger.java index 31450cd..c879b12 100644 --- a/app/src/main/java/com/rubenvandeven/emotionhero/ScenarioAnger.java +++ b/app/src/main/java/com/rubenvandeven/emotionhero/ScenarioAnger.java @@ -7,6 +7,11 @@ import android.util.Log; */ public class ScenarioAnger extends Scenario { + + public ScenarioAnger(GamingActivity activity) { + super(activity); + } + void createScenario() { Log.d(GamingActivity.LOG_TAG, "CREATE SCENARIO: anger"); diff --git a/app/src/main/java/com/rubenvandeven/emotionhero/ScenarioView.java b/app/src/main/java/com/rubenvandeven/emotionhero/ScenarioView.java index 608d7f3..815b06e 100644 --- a/app/src/main/java/com/rubenvandeven/emotionhero/ScenarioView.java +++ b/app/src/main/java/com/rubenvandeven/emotionhero/ScenarioView.java @@ -6,8 +6,7 @@ import android.graphics.Color; import android.graphics.Paint; import android.graphics.Path; import android.graphics.PixelFormat; -import android.util.AttributeSet; -import android.util.Log; +import android.graphics.Typeface; import android.view.SurfaceHolder; import android.view.SurfaceView; @@ -32,6 +31,14 @@ public class ScenarioView extends SurfaceView implements SurfaceHolder.Callback */ private Map currentAttributeScores = new HashMap<>(); + //avoid object instantiation on each onDraw + private Map emoPaints = new HashMap<>(); + private Map emoOutlinePaints = new HashMap<>(); + private Map emoScoredPaints = new HashMap<>(); + private Paint mainPaint = new Paint(); + private Paint attrScorePaint = new Paint(); + private Paint linePaint = new Paint(); + // see: http://blog.danielnadeau.io/2012/01/android-canvas-beginners-tutorial.html class PanelThread extends Thread { @@ -58,6 +65,7 @@ public class ScenarioView extends SurfaceView implements SurfaceHolder.Callback c = null; //set to false and loop ends, stopping thread try { +// c = _surfaceHolder.lockCanvas(null); c = _surfaceHolder.lockCanvas(null); synchronized (_surfaceHolder) { //Insert methods to modify positions of items in onDraw() @@ -76,6 +84,43 @@ public class ScenarioView extends SurfaceView implements SurfaceHolder.Callback super(context); getHolder().addCallback(this); _scenario = s; + + //setup paints for drawing + mainPaint.setColor(Color.GRAY); + mainPaint.setTextSize(40); + mainPaint.setTypeface(Typeface.SANS_SERIF); + + linePaint.setColor(Color.GRAY); + linePaint.setStrokeWidth(5); + attrScorePaint.setColor(Color.DKGRAY); + + for(Emotion emotion: Emotion.values()) { + Paint emoPaint = new Paint(); + emoPaint.setTextSize(20); + emoPaint.setColor(emotion.getColor()); + emoPaints.put(emotion, emoPaint); + + Paint emoPaintOutline = new Paint(); + emoPaintOutline.setColor(emotion.getColor()); + emoPaintOutline.setStyle(Paint.Style.STROKE); + emoPaintOutline.setStrokeWidth(2); + emoOutlinePaints.put(emotion, emoPaintOutline); + + Paint emoScoredPaint = new Paint(); + + float darkenFactor = 0.4f; + int red = (int) ((Color.red(emotion.getColor()) * darkenFactor)); + int green = (int) ((Color.green(emotion.getColor()) * darkenFactor)); + int blue = (int) ((Color.blue(emotion.getColor()) * darkenFactor)); + int emoLightenedColor = Color.rgb(red, green, blue); + + emoScoredPaint.setColor(emoLightenedColor); + emoScoredPaint.setTextSize(20); + emoScoredPaint.setStrokeWidth(2); + emoScoredPaint.setStyle(Paint.Style.FILL_AND_STROKE); + emoScoredPaints.put(emotion, emoScoredPaint); + + } } @@ -96,15 +141,6 @@ public class ScenarioView extends SurfaceView implements SurfaceHolder.Callback float step_y = (canvas.getWidth() * used_width) / Emotion.values().length; float max_ball_radius = step_y * (float) 0.8 / 2; - -// bottom at 80%; - Paint mainPaint = new Paint(); - mainPaint.setColor(Color.GRAY); - - - Paint linePaint = new Paint(); - linePaint.setColor(Color.GRAY); - linePaint.setStrokeWidth(5); // canvas.drawLine(0, bottomline_height, width, bottomline_height, linePaint); @@ -117,22 +153,16 @@ public class ScenarioView extends SurfaceView implements SurfaceHolder.Callback if(value < 5) { value = 5; } - Paint emoPaint = new Paint(); - Paint emoPaintOutline = new Paint(); - emoPaint.setTextSize(20); - emoPaint.setColor(emotion.getColor()); - emoPaintOutline.setColor(emotion.getColor()); - emoPaintOutline.setStyle(Paint.Style.STROKE); - emoPaintOutline.setStrokeWidth(2); float cx = padding_left + (step_y * emotion.ordinal() + step_y / 2); float cy = bottomline_height; canvas.drawCircle(cx, cy, max_ball_radius, mainPaint); - canvas.drawCircle(cx, cy, max_ball_radius, emoPaintOutline); + canvas.drawCircle(cx, cy, max_ball_radius, emoOutlinePaints.get(emotion)); - canvas.drawCircle(cx, cy, max_ball_radius * value/100, emoPaint); +// canvas.drawCircle(cx, cy, max_ball_radius * value/100, emoPaints.get(emotion.ordinal())); + canvas.drawCircle(cx, cy, max_ball_radius * value/100, attrScorePaint); Path emoNamePath = new Path(); emoNamePath.moveTo(cx, cy + max_ball_radius * 1.5f); @@ -141,7 +171,7 @@ public class ScenarioView extends SurfaceView implements SurfaceHolder.Callback emoNamePath.rLineTo(1000,1000); // canvas.drawText(emotion.toString(), cx, cy + max_ball_radius * (float) 1.3, emoPaint); - canvas.drawTextOnPath(emotion.toString(), emoNamePath, 0, 0, emoPaint); + canvas.drawTextOnPath(emotion.toString(), emoNamePath, 0, 0, emoPaints.get(emotion)); } // Draw targets: @@ -150,21 +180,28 @@ public class ScenarioView extends SurfaceView implements SurfaceHolder.Callback float diff_y = sec_height * _scenario.getTime(); for(Scenario.Target target: _scenario.getTargets()) { - Paint emoPaint = new Paint(); - emoPaint.setColor(target.emotion.getColor()); - + Paint targetPaint = (target.score == null) ? emoPaints.get(target.emotion) : emoScoredPaints.get(target.emotion); float cy = bottomline_height - (target.timestamp * sec_height) + diff_y; float cx = padding_left + (step_y * target.emotion.ordinal() + step_y / 2); - canvas.drawCircle(cx, cy, max_ball_radius * target.value/100, emoPaint); + canvas.drawCircle(cx, cy, max_ball_radius * target.value/100, targetPaint); + + if(target.score != null) { + canvas.drawText(Float.toString(target.score.value), cx, cy, targetPaint); + } + // String target_text = target.emotion.toString() + " " + target.value + "%"; // canvas.drawText(target_text, cx, y_pos + diff_y , emoPaint); } + canvas.drawText("Total: " + Float.toString(_scenario.getTotalScore()), 50, 50, mainPaint); + + } +// TODO: create AttributeScoreCollection class, with this method. public void setCurrentAttributeScoresForFace(Face face) { for(Emotion emotion: Emotion.values()) { diff --git a/app/src/main/res/raw/score2.mp3 b/app/src/main/res/raw/score2.mp3 new file mode 100644 index 0000000..489b730 Binary files /dev/null and b/app/src/main/res/raw/score2.mp3 differ