Add sound and proper tick for scenario

app/src/main/java/com/rubenvandeven/emotionhero/GamingActivity.java

@@ -2,9 +2,12 @@ package com.rubenvandeven.emotionhero;
 
 import android.Manifest;
 import android.annotation.SuppressLint;
+import android.annotation.TargetApi;
 import android.content.pm.PackageManager;
-import android.graphics.Bitmap;
+import android.media.AudioAttributes;
-import android.graphics.Canvas;
+import android.media.AudioManager;
+import android.media.SoundPool;
+import android.os.Build;
 import android.support.v4.app.ActivityCompat;
 import android.support.v7.app.ActionBar;
 import android.support.v7.app.AppCompatActivity;
@@ -25,6 +28,7 @@ import com.affectiva.android.affdex.sdk.detector.CameraDetector;
 import com.affectiva.android.affdex.sdk.detector.Detector;
 import com.affectiva.android.affdex.sdk.detector.Face;
 
+import java.util.HashMap;
 import java.util.List;
 
 /**
@@ -123,6 +127,12 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL
 
     TextView paramText;
 
+    public SoundPool sound;
+    public HashMap<Integer, Integer> soundIds = new HashMap<>();
+
+    final static int SOUND_SCORE = 1;
+
+
     @Override
     protected void onCreate(Bundle savedInstanceState) {
         super.onCreate(savedInstanceState);
@@ -182,6 +192,7 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL
                     }
                 }
                 setMeasuredDimension(width,height);
+//                setMeasuredDimension(1,1); // this DOES increase performance....
             }
         };
         RelativeLayout.LayoutParams params = new RelativeLayout.LayoutParams(ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.WRAP_CONTENT);
@@ -191,16 +202,15 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL
         videoLayout.addView(cameraPreview,0);
 
 
-        currentScenario = new ScenarioAnger();
+        currentScenario = new ScenarioAnger(this);
 
         scenarioView = new ScenarioView(this, currentScenario);
         RelativeLayout.LayoutParams scenarioViewParams = new RelativeLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT);
         videoLayout.addView(scenarioView, 1, scenarioViewParams);
-//        new LayoutParams(LayoutParams.FILL_PARENT, LayoutParams.WRAP_CONTENT)
-//        videoLayout.addView(scenarioView, 200, 100);
 
+        createSoundPool(); // instantiate SoundPool in sound
+        soundIds.put(SOUND_SCORE, sound.load(this, R.raw.score2, 1));
 
-//        startDetector();
     }
 
     @Override
@@ -254,14 +264,16 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL
                 if(detector == null)
                 {
 //                    SurfaceView surfaceView = (SurfaceView) findViewById(R.id.surfaceView);
-                    detector = new CameraDetector(this, CameraDetector.CameraType.CAMERA_FRONT, cameraPreview);
+
+                    detector = new CameraDetector(this, CameraDetector.CameraType.CAMERA_FRONT, cameraPreview, 1, Detector.FaceDetectorMode.LARGE_FACES);
                     detector.setLicensePath("emotionhero_dev.license");
+
                     detector.setMaxProcessRate(10);
                     detector.setDetectAllEmotions(true);
                     detector.setDetectAllAppearances(false);
                     detector.setDetectAllEmojis(false);
                     detector.setDetectAllExpressions(false);
-                    detector.setMaxProcessRate(12);
+                    detector.setMaxProcessRate(20);
 
                     detector.setImageListener(this);
                     detector.setOnCameraEventListener(this);
@@ -269,7 +281,7 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL
                 }
 
                 detector.start();
-                mContentView.setText("STARTING...");
+                setText("STARTING...");
                 Log.d(LOG_TAG, Boolean.toString(detector.isRunning()));
             }
         }
@@ -321,10 +333,12 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL
      * Detector callback gives the faces found so we can match their scores to the given scenario.
      */
     public void onImageResults(List<Face> list, Frame frame, float timestamp) {
+//        frame.getOriginalBitmapFrame()
 //        Log.e(LOG_TAG, "RESULT! faces: " + Integer.toString(list.size()) + " t: " + Float.toString(timestamp) + "s" );
-        if(!currentScenario.isWithinTime(timestamp))
+//        if(!currentScenario.isWithinTime(timestamp))
+        if(currentScenario.isFinished())
         {
-            mContentView.setText(String.format("LEVEL ENDED\nScore: %.2f", currentScenario.getTotalScore()));
+            setText(String.format("LEVEL ENDED\nScore: %.2f", currentScenario.getTotalScore()));
             stopDetector();
             restartButton.setVisibility(View.VISIBLE);
             return;
@@ -332,12 +346,12 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL
         if (list == null)
             return;
         if (list.size() == 0) {
-            mContentView.setText("NO FACE FOUND");
+//            mContentView.setText("NO FACE FOUND"); // this happens in onFaceDetectionStopped
         } else {
+            hideText(); // hide textView as we want as few elements as possible.
             Face face = list.get(0);
-            currentScenario.validateFaceOnTime(face, timestamp);
+            currentScenario.setCurrentAttributeScoresForFace(face);
             scenarioView.setCurrentAttributeScoresForFace(face);
-            mContentView.setText(String.format("SCORE \n%.2f",currentScenario.getTotalScore()));
 
 //            String paramString = "";
 //            paramString += "Anger    " + String.format("%02.2f", face.emotions.getAnger()) + "%\n";
@@ -394,14 +408,53 @@ public class GamingActivity extends AppCompatActivity implements Detector.ImageL
     @Override
     public void onFaceDetectionStarted()
     {
-        mContentView.setText("START!");
+        setText("START!");
         currentScenario.start();
     }
 
     @Override
     public void onFaceDetectionStopped()
     {
-        mContentView.setText("No face found...");
+        setText("No face found...");
-//        paramText.setText("No face found");
+        currentScenario.pause();
+    }
+
+    public void setText(String text)
+    {
+        mContentView.setVisibility(View.VISIBLE);
+        mContentView.setText(text);
+    }
+
+    public void hideText()
+    {
+        mContentView.setVisibility(View.GONE);
+    }
+
+
+//    http://stackoverflow.com/a/27552576
+    protected void createSoundPool() {
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
+            createNewSoundPool();
+        } else {
+            createOldSoundPool();
+        }
+    }
+
+//    http://stackoverflow.com/a/27552576
+    @TargetApi(Build.VERSION_CODES.LOLLIPOP)
+    protected void createNewSoundPool(){
+        AudioAttributes attributes = new AudioAttributes.Builder()
+                .setUsage(AudioAttributes.USAGE_GAME)
+                .setContentType(AudioAttributes.CONTENT_TYPE_SONIFICATION)
+                .build();
+        sound = new SoundPool.Builder()
+                .setAudioAttributes(attributes)
+                .build();
+    }
+
+//    http://stackoverflow.com/a/27552576
+    @SuppressWarnings("deprecation")
+    protected void createOldSoundPool(){
+        sound = new SoundPool(5, AudioManager.STREAM_MUSIC,0);
     }
 }

app/src/main/java/com/rubenvandeven/emotionhero/Scenario.java

@@ -1,16 +1,12 @@
 package com.rubenvandeven.emotionhero;
 
-import android.graphics.Bitmap;
-import android.graphics.Canvas;
-import android.graphics.Color;
-import android.graphics.Paint;
-import android.util.Log;
-
 import com.affectiva.android.affdex.sdk.detector.Face;
 
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
 
 /**
  * Created by ruben on 16/08/16.
@@ -18,10 +14,36 @@ import java.util.Map;
 
 abstract public class Scenario {
 
+    static int DESIRED_FPS = 25;
+
     float duration = 0;
 
+    /**
+     * @deprecated
+     */
     long startTime = 0;
 
+    /**
+     * The timer that provides the tick
+     */
+    Timer timer;
+
+    /**
+     * Increment on each tick
+     */
+    float runningTime = -1;
+
+
+    boolean isRunning = false;
+
+    private GamingActivity _activity;
+
+    /**
+     * The scorres in this moment, as to draw them on the screen.
+     * Indexes are Emotion ordinals
+     */
+    private Map<Emotion, Float> currentAttributeScores = new HashMap<>();
+
     ArrayList<Target> targets = new ArrayList<>();
 
     abstract void createScenario();
@@ -30,6 +52,7 @@ abstract public class Scenario {
         public Emotion emotion;
         public float value;
         public float timestamp;
+        public Score score;
     }
 
     ArrayList<Score> scores = new ArrayList<>();
@@ -40,20 +63,68 @@ abstract public class Scenario {
          * Extra bonus given
          */
         public boolean isSpotOn = false;
-        /**
-         * The target the score is awarded for
-         */
-        public Target target;
     }
 
     /**
      * Constructor
      */
-    public Scenario()
+    public Scenario(GamingActivity activity)
     {
         createScenario();
+        timer = new Timer("ScenarioTimer");
+        TimerTask tickTask;
+        tickTask = new TimerTask() {
+            @Override
+            public void run() {
+                //                if (System.currentTimeMillis() - scheduledExecutionTime() >=
+                //                        MAX_TARDINESS)
+                //                    return;  // Too late; skip this execution.
+                tick();
+            }
+        };
+        timer.schedule(tickTask, 0, 1000/DESIRED_FPS);
+        _activity = activity;
     }
 
+    /**
+     * To be called on each timer tick
+     */
+    public void tick()
+    {
+        if(!isRunning)
+            return;
+
+        runningTime += 1.0f/DESIRED_FPS;
+
+        if(isFinished()) {
+            stop();
+            return;
+        }
+
+        for (int i = targets.size() - 1; i >= 0; i--) {
+            Target target = targets.get(i);
+            // skip targets that are already scored
+            if(target.score != null) {
+                continue;
+            }
+            if(target.timestamp <= runningTime) {
+                float scored_value = currentAttributeScores.get(target.emotion);
+                float required_value = target.value;
+                Score score = new Score();
+                score.value = Math.round(100 - Math.abs(scored_value-required_value));
+                scores.add(score);
+
+                //
+                _activity.sound.play(_activity.soundIds.get(_activity.SOUND_SCORE), 1, 1,
+                                        1,0, score.value / 200f + 0.5f ); // play back the sound slower
+                                                                        // depending on score value
+                target.score = score;
+            }
+        }
+
+    }
+
+
     /**
      * Add a target on given timestamp
      * @param emotion
@@ -63,9 +134,9 @@ abstract public class Scenario {
     public void setTarget(Emotion emotion, float value, float timestamp)
     {
 //        Log.e(GamingActivity.LOG_TAG, "Set target:" + Float.toString(timestamp) + " " + Float.toString(duration));
-        if(timestamp > duration)
+        if((timestamp + 1) > duration)
         {
-            duration = timestamp;
+            duration = timestamp + 1; // always a bit onger than last target, as it otherwise the game does not finish pretty
         }
         Target target = new Target();
         target.timestamp = timestamp;
@@ -91,6 +162,11 @@ abstract public class Scenario {
         setTarget(emotion, value, timestamp);
     }
 
+    /**
+     * @deprecated use @see tick()
+     * @param face
+     * @param timestamp
+     */
     public void validateFaceOnTime(Face face, float timestamp)
     {
 //        TODO: interpolation of time
@@ -101,7 +177,7 @@ abstract public class Scenario {
                 float required_value = target.value;
                 Score score = new Score();
                 score.value = 100 - Math.abs(scored_value-required_value);
-                score.target = target;
+                target.score = score;
                 scores.add(score);
             }
         }
@@ -135,21 +211,46 @@ abstract public class Scenario {
      * Get the time within the scenario (so since start() has been called)
      */
     public float getTime() {
+        return runningTime;
         // if not started, don't move the labels, if started, move them by diff_y
-        if(startTime == 0) {
+//        if(startTime == 0) {
-            return 0;
+//            return 0;
-        } else {
+//        } else {
-            float diff_t = ((System.currentTimeMillis() - startTime)) / (float) 1000;
+//            float diff_t = ((System.currentTimeMillis() - startTime)) / (float) 1000;
-            if(diff_t > duration) { // never larger than scenario duration
+//            if(diff_t > duration) { // never larger than scenario duration
-                return duration;
+//                return duration;
-            }
+//            }
-            return diff_t;
+//            return diff_t;
-        }
+//        }
     }
 
     public void start()
     {
         startTime = System.currentTimeMillis();
+        isRunning = true;
+    }
+
+    public void pause() {
+        isRunning = false;
+    }
+
+    public void stop()
+    {
+        isRunning = false;
+    }
+
+
+    //  TODO: create AttributeScoreCollection class, with this method.
+    public void setCurrentAttributeScoresForFace(Face face)
+    {
+        for(Emotion emotion: Emotion.values()) {
+            currentAttributeScores.put(emotion, emotion.getValueFromFace(face));
+        }
+    }
+
+    public boolean isFinished()
+    {
+        return runningTime > duration;
     }
 
     // TODO: create a 'tick' that checks all current values with requirements and increases the timer etc

app/src/main/java/com/rubenvandeven/emotionhero/ScenarioAnger.java

@@ -7,6 +7,11 @@ import android.util.Log;
  */
 
 public class ScenarioAnger extends Scenario {
+
+    public ScenarioAnger(GamingActivity activity) {
+        super(activity);
+    }
+
     void createScenario()
     {
         Log.d(GamingActivity.LOG_TAG, "CREATE SCENARIO: anger");

app/src/main/java/com/rubenvandeven/emotionhero/ScenarioView.java

@@ -6,8 +6,7 @@ import android.graphics.Color;
 import android.graphics.Paint;
 import android.graphics.Path;
 import android.graphics.PixelFormat;
-import android.util.AttributeSet;
+import android.graphics.Typeface;
-import android.util.Log;
 import android.view.SurfaceHolder;
 import android.view.SurfaceView;
 
@@ -32,6 +31,14 @@ public class ScenarioView extends SurfaceView implements SurfaceHolder.Callback
      */
     private Map<Emotion, Float> currentAttributeScores = new HashMap<>();
 
+    //avoid object instantiation on each onDraw
+    private Map<Emotion, Paint> emoPaints = new HashMap<>();
+    private Map<Emotion, Paint> emoOutlinePaints = new HashMap<>();
+    private Map<Emotion, Paint> emoScoredPaints = new HashMap<>();
+    private Paint mainPaint = new Paint();
+    private Paint attrScorePaint = new Paint();
+    private Paint linePaint = new Paint();
+
 
     // see: http://blog.danielnadeau.io/2012/01/android-canvas-beginners-tutorial.html
     class PanelThread extends Thread {
@@ -58,6 +65,7 @@ public class ScenarioView extends SurfaceView implements SurfaceHolder.Callback
                 c = null;      //set to false and loop ends, stopping thread
 
                 try {
+//                    c = _surfaceHolder.lockCanvas(null);
                     c = _surfaceHolder.lockCanvas(null);
                     synchronized (_surfaceHolder) {
                         //Insert methods to modify positions of items in onDraw()
@@ -76,6 +84,43 @@ public class ScenarioView extends SurfaceView implements SurfaceHolder.Callback
         super(context);
         getHolder().addCallback(this);
         _scenario = s;
+
+        //setup paints for drawing
+        mainPaint.setColor(Color.GRAY);
+        mainPaint.setTextSize(40);
+        mainPaint.setTypeface(Typeface.SANS_SERIF);
+
+        linePaint.setColor(Color.GRAY);
+        linePaint.setStrokeWidth(5);
+        attrScorePaint.setColor(Color.DKGRAY);
+
+        for(Emotion emotion: Emotion.values()) {
+            Paint emoPaint = new Paint();
+            emoPaint.setTextSize(20);
+            emoPaint.setColor(emotion.getColor());
+            emoPaints.put(emotion, emoPaint);
+
+            Paint emoPaintOutline = new Paint();
+            emoPaintOutline.setColor(emotion.getColor());
+            emoPaintOutline.setStyle(Paint.Style.STROKE);
+            emoPaintOutline.setStrokeWidth(2);
+            emoOutlinePaints.put(emotion, emoPaintOutline);
+
+            Paint emoScoredPaint = new Paint();
+
+            float darkenFactor = 0.4f;
+            int red = (int) ((Color.red(emotion.getColor()) * darkenFactor));
+            int green = (int) ((Color.green(emotion.getColor()) * darkenFactor));
+            int blue = (int) ((Color.blue(emotion.getColor()) * darkenFactor));
+            int emoLightenedColor = Color.rgb(red, green, blue);
+
+            emoScoredPaint.setColor(emoLightenedColor);
+            emoScoredPaint.setTextSize(20);
+            emoScoredPaint.setStrokeWidth(2);
+            emoScoredPaint.setStyle(Paint.Style.FILL_AND_STROKE);
+            emoScoredPaints.put(emotion, emoScoredPaint);
+
+        }
     }
 
 
@@ -96,15 +141,6 @@ public class ScenarioView extends SurfaceView implements SurfaceHolder.Callback
         float step_y = (canvas.getWidth() * used_width) / Emotion.values().length;
         float max_ball_radius = step_y * (float) 0.8 / 2;
 
-
-//        bottom at 80%;
-        Paint mainPaint = new Paint();
-        mainPaint.setColor(Color.GRAY);
-
-
-        Paint linePaint = new Paint();
-        linePaint.setColor(Color.GRAY);
-        linePaint.setStrokeWidth(5);
 //        canvas.drawLine(0, bottomline_height, width, bottomline_height, linePaint);
 
 
@@ -117,22 +153,16 @@ public class ScenarioView extends SurfaceView implements SurfaceHolder.Callback
             if(value < 5) {
                 value = 5;
             }
-            Paint emoPaint = new Paint();
-            Paint emoPaintOutline = new Paint();
-            emoPaint.setTextSize(20);
-            emoPaint.setColor(emotion.getColor());
-            emoPaintOutline.setColor(emotion.getColor());
-            emoPaintOutline.setStyle(Paint.Style.STROKE);
-            emoPaintOutline.setStrokeWidth(2);
 
             float cx = padding_left + (step_y * emotion.ordinal() + step_y / 2);
             float cy = bottomline_height;
 
 
             canvas.drawCircle(cx, cy, max_ball_radius, mainPaint);
-            canvas.drawCircle(cx, cy, max_ball_radius, emoPaintOutline);
+            canvas.drawCircle(cx, cy, max_ball_radius, emoOutlinePaints.get(emotion));
 
-            canvas.drawCircle(cx, cy, max_ball_radius * value/100, emoPaint);
+//            canvas.drawCircle(cx, cy, max_ball_radius * value/100, emoPaints.get(emotion.ordinal()));
+            canvas.drawCircle(cx, cy, max_ball_radius * value/100, attrScorePaint);
 
             Path emoNamePath = new Path();
             emoNamePath.moveTo(cx, cy + max_ball_radius * 1.5f);
@@ -141,7 +171,7 @@ public class ScenarioView extends SurfaceView implements SurfaceHolder.Callback
             emoNamePath.rLineTo(1000,1000);
 
 //            canvas.drawText(emotion.toString(), cx, cy + max_ball_radius * (float) 1.3, emoPaint);
-            canvas.drawTextOnPath(emotion.toString(), emoNamePath, 0, 0, emoPaint);
+            canvas.drawTextOnPath(emotion.toString(), emoNamePath, 0, 0, emoPaints.get(emotion));
         }
 
         // Draw targets:
@@ -150,21 +180,28 @@ public class ScenarioView extends SurfaceView implements SurfaceHolder.Callback
         float diff_y = sec_height * _scenario.getTime();
 
         for(Scenario.Target target: _scenario.getTargets()) {
-            Paint emoPaint = new Paint();
+            Paint targetPaint = (target.score == null) ? emoPaints.get(target.emotion) : emoScoredPaints.get(target.emotion);
-            emoPaint.setColor(target.emotion.getColor());
-
             float cy = bottomline_height - (target.timestamp * sec_height) + diff_y;
 
             float cx = padding_left + (step_y * target.emotion.ordinal() + step_y / 2);
 
-            canvas.drawCircle(cx, cy, max_ball_radius * target.value/100, emoPaint);
+            canvas.drawCircle(cx, cy, max_ball_radius * target.value/100, targetPaint);
+
+            if(target.score != null) {
+                canvas.drawText(Float.toString(target.score.value), cx, cy, targetPaint);
+            }
+
 
 //            String target_text = target.emotion.toString() + " " + target.value + "%";
 //            canvas.drawText(target_text, cx, y_pos + diff_y , emoPaint);
         }
 
+        canvas.drawText("Total: " + Float.toString(_scenario.getTotalScore()), 50, 50, mainPaint);
+
+
     }
 
+//  TODO: create AttributeScoreCollection class, with this method.
     public void setCurrentAttributeScoresForFace(Face face)
     {
         for(Emotion emotion: Emotion.values()) {