commit 57974468dfb73ffbb1c68bf2cab39a54e316181f Author: Ruben Date: Tue Aug 16 19:25:28 2016 +0200 Initial 'game' setup diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2c4ef38 --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +*.iml +.gradle +/local.properties +/.idea/workspace.xml +/.idea/libraries +.DS_Store +/build +/captures +.externalNativeBuild +*.license + diff --git a/.idea/compiler.xml b/.idea/compiler.xml new file mode 100644 index 0000000..96cc43e --- /dev/null +++ b/.idea/compiler.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/copyright/profiles_settings.xml b/.idea/copyright/profiles_settings.xml new file mode 100644 index 0000000..e7bedf3 --- /dev/null +++ b/.idea/copyright/profiles_settings.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/.idea/gradle.xml b/.idea/gradle.xml new file mode 100644 index 0000000..0e23f8e --- /dev/null +++ b/.idea/gradle.xml @@ -0,0 +1,19 @@ + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..bd04605 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,62 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Android API 23 Platform + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..ac940f8 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/runConfigurations.xml b/.idea/runConfigurations.xml new file mode 100644 index 0000000..7f68460 --- /dev/null +++ b/.idea/runConfigurations.xml @@ -0,0 +1,12 @@ + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/app/.gitignore b/app/.gitignore new file mode 100644 index 0000000..796b96d --- /dev/null +++ b/app/.gitignore @@ -0,0 +1 @@ +/build diff --git a/app/build.gradle b/app/build.gradle new file mode 100644 index 0000000..d0ad463 --- /dev/null +++ b/app/build.gradle @@ -0,0 +1,31 @@ +apply plugin: 'com.android.application' + +android { + compileSdkVersion 23 + buildToolsVersion '23.0.1' + defaultConfig { + applicationId "com.rubenvandeven.emotionhero" + minSdkVersion 16 + targetSdkVersion 23 + versionCode 1 + versionName "1.0" + testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner" + } + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro' + } + } +} + +dependencies { + compile fileTree(include: ['*.jar'], dir: 'libs') + androidTestCompile('com.android.support.test.espresso:espresso-core:2.2.2', { + exclude group: 'com.android.support', module: 'support-annotations' + }) + compile 'com.android.support:appcompat-v7:23.4.0' + compile 'com.android.support:support-v4:23.4.0' + compile 'com.affectiva.android:affdexsdk:3.1' + testCompile 'junit:junit:4.12' +} diff --git a/app/proguard-rules.pro b/app/proguard-rules.pro new file mode 100644 index 0000000..77e424f --- /dev/null +++ b/app/proguard-rules.pro @@ -0,0 +1,17 @@ +# Add project specific ProGuard rules here. +# By default, the flags in this file are appended to flags specified +# in /home/ruben/Android/Sdk/tools/proguard/proguard-android.txt +# You can edit the include path and order by changing the proguardFiles +# directive in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# Add any project specific keep options here: + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} diff --git a/app/src/androidTest/java/com/rubenvandeven/emotionhero/ExampleInstrumentedTest.java b/app/src/androidTest/java/com/rubenvandeven/emotionhero/ExampleInstrumentedTest.java new file mode 100644 index 0000000..5cf4773 --- /dev/null +++ b/app/src/androidTest/java/com/rubenvandeven/emotionhero/ExampleInstrumentedTest.java @@ -0,0 +1,26 @@ +package com.rubenvandeven.emotionhero; + +import android.content.Context; +import android.support.test.InstrumentationRegistry; +import android.support.test.runner.AndroidJUnit4; + +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.junit.Assert.*; + +/** + * Instrumentation test, which will execute on an Android device. + * + * @see Testing documentation + */ +@RunWith(AndroidJUnit4.class) +public class ExampleInstrumentedTest { + @Test + public void useAppContext() throws Exception { + // Context of the app under test. + Context appContext = InstrumentationRegistry.getTargetContext(); + + assertEquals("com.rubenvandeven.emotionhero", appContext.getPackageName()); + } +} diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml new file mode 100644 index 0000000..6a30b6b --- /dev/null +++ b/app/src/main/AndroidManifest.xml @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/app/src/main/java/com/rubenvandeven/emotionhero/Emotion.java b/app/src/main/java/com/rubenvandeven/emotionhero/Emotion.java new file mode 100644 index 0000000..57df030 --- /dev/null +++ b/app/src/main/java/com/rubenvandeven/emotionhero/Emotion.java @@ -0,0 +1,38 @@ +package com.rubenvandeven.emotionhero; + +import com.affectiva.android.affdex.sdk.detector.Face; + +/** + * Created by ruben on 16/08/16. + */ + +public enum Emotion { + ANGER, + CONTEMPT, + DISGUST, + FEAR, + JOY, + SADNESS, + SURPRISE; + + public float getValueFromFace(Face face) + { + switch (this) { + case ANGER: + return face.emotions.getAnger(); + case CONTEMPT: + return face.emotions.getContempt(); + case DISGUST: + return face.emotions.getDisgust(); + case FEAR: + return face.emotions.getFear(); + case JOY: + return face.emotions.getJoy(); + case SADNESS: + return face.emotions.getSadness(); + case SURPRISE: + return face.emotions.getSurprise(); + } + return 0; + } +} diff --git a/app/src/main/java/com/rubenvandeven/emotionhero/GamingActivity.java b/app/src/main/java/com/rubenvandeven/emotionhero/GamingActivity.java new file mode 100644 index 0000000..2490e2a --- /dev/null +++ b/app/src/main/java/com/rubenvandeven/emotionhero/GamingActivity.java @@ -0,0 +1,361 @@ +package com.rubenvandeven.emotionhero; + +import android.Manifest; +import android.annotation.SuppressLint; +import android.content.pm.PackageManager; +import android.graphics.Bitmap; +import android.graphics.Canvas; +import android.support.v4.app.ActivityCompat; +import android.support.v7.app.ActionBar; +import android.support.v7.app.AppCompatActivity; +import android.os.Bundle; +import android.os.Handler; +import android.util.Log; +import android.view.MotionEvent; +import android.view.SurfaceView; +import android.view.View; +import android.view.ViewGroup; +import android.widget.RelativeLayout; +import android.widget.TextView; +import android.widget.Toast; + +import com.affectiva.android.affdex.sdk.Frame; +import com.affectiva.android.affdex.sdk.detector.CameraDetector; +import com.affectiva.android.affdex.sdk.detector.Detector; +import com.affectiva.android.affdex.sdk.detector.Face; + +import java.util.List; + +/** + * An example full-screen activity that shows and hides the system UI (i.e. + * status bar and navigation/system bar) with user interaction. + */ +public class GamingActivity extends AppCompatActivity implements Detector.ImageListener, CameraDetector.CameraEventListener, Detector.FaceListener { + + final String LOG_TAG = "EmotionHero"; + + final int PERMISSIONS_REQUEST_CAMERA = 1; + + /** + * Whether or not the system UI should be auto-hidden after + * {@link #AUTO_HIDE_DELAY_MILLIS} milliseconds. + */ + private static final boolean AUTO_HIDE = true; + + /** + * If {@link #AUTO_HIDE} is set, the number of milliseconds to wait after + * user interaction before hiding the system UI. + */ + private static final int AUTO_HIDE_DELAY_MILLIS = 3000; + + /** + * Some older devices needs a small delay between UI widget updates + * and a change of the status and navigation bar. + */ + private static final int UI_ANIMATION_DELAY = 300; + private final Handler mHideHandler = new Handler(); + private TextView mContentView; + private final Runnable mHidePart2Runnable = new Runnable() { + @SuppressLint("InlinedApi") + @Override + public void run() { + // Delayed removal of status and navigation bar + + // Note that some of these constants are new as of API 16 (Jelly Bean) + // and API 19 (KitKat). It is safe to use them, as they are inlined + // at compile-time and do nothing on earlier devices. + mContentView.setSystemUiVisibility(View.SYSTEM_UI_FLAG_LOW_PROFILE + | View.SYSTEM_UI_FLAG_FULLSCREEN + | View.SYSTEM_UI_FLAG_LAYOUT_STABLE + | View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY + | View.SYSTEM_UI_FLAG_LAYOUT_HIDE_NAVIGATION + | View.SYSTEM_UI_FLAG_HIDE_NAVIGATION); + } + }; + private View mControlsView; + private final Runnable mShowPart2Runnable = new Runnable() { + @Override + public void run() { + // Delayed display of UI elements + ActionBar actionBar = getSupportActionBar(); + if (actionBar != null) { + actionBar.show(); + } + mControlsView.setVisibility(View.VISIBLE); + } + }; + private boolean mVisible; + private final Runnable mHideRunnable = new Runnable() { + @Override + public void run() { + hide(); + } + }; + /** + * Touch listener to use for in-layout UI controls to delay hiding the + * system UI. This is to prevent the jarring behavior of controls going away + * while interacting with activity UI. + */ + private final View.OnTouchListener mDelayHideTouchListener = new View.OnTouchListener() { + @Override + public boolean onTouch(View view, MotionEvent motionEvent) { + if (AUTO_HIDE) { + delayedHide(AUTO_HIDE_DELAY_MILLIS); + } + return false; + } + }; + + private CameraDetector detector; + + SurfaceView cameraPreview; + + int previewWidth = 0; + int previewHeight = 0; + + Scenario currentScenario; + + boolean has_camera_permission = false; + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + + setContentView(R.layout.activity_gaming); + + mVisible = true; + mControlsView = findViewById(R.id.fullscreen_content_controls); + mContentView = (TextView) findViewById(R.id.fullscreen_content); + RelativeLayout videoLayout = (RelativeLayout) findViewById(R.id.video_layout); + + + // Set up the user interaction to manually show or hide the system UI. + mContentView.setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View view) { + toggle(); + } + }); + + // Upon interacting with UI controls, delay any scheduled hide() + // operations to prevent the jarring behavior of controls going away + // while interacting with the UI. + findViewById(R.id.dummy_button).setOnTouchListener(mDelayHideTouchListener); + + //We create a custom SurfaceView that resizes itself to match the aspect ratio of the incoming camera frames + cameraPreview = new SurfaceView(this) { + @Override + public void onMeasure(int widthSpec, int heightSpec) { + int measureWidth = MeasureSpec.getSize(widthSpec); + int measureHeight = MeasureSpec.getSize(heightSpec); + int width; + int height; + if (previewHeight == 0 || previewWidth == 0) { + width = measureWidth; + height = measureHeight; + } else { + float viewAspectRatio = (float)measureWidth/measureHeight; + float cameraPreviewAspectRatio = (float) previewWidth/previewHeight; + + if (cameraPreviewAspectRatio > viewAspectRatio) { + width = measureWidth; + height =(int) (measureWidth / cameraPreviewAspectRatio); + } else { + width = (int) (measureHeight * cameraPreviewAspectRatio); + height = measureHeight; + } + } + setMeasuredDimension(width,height); + } + }; + RelativeLayout.LayoutParams params = new RelativeLayout.LayoutParams(ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.WRAP_CONTENT); + params.addRule(RelativeLayout.CENTER_IN_PARENT,RelativeLayout.TRUE); + cameraPreview.setLayoutParams(params); + videoLayout.addView(cameraPreview,0); + + currentScenario = new ScenarioAnger(); + +// startDetector(); + } + + @Override + protected void onPostCreate(Bundle savedInstanceState) { + super.onPostCreate(savedInstanceState); + + // Trigger the initial hide() shortly after the activity has been + // created, to briefly hint to the user that UI controls + // are available. + delayedHide(100); + } + + private void toggle() { + if (mVisible) { + hide(); + } else { + show(); + } + } + + @Override + protected void onResume() { + super.onResume(); + startDetector(); + + } + + @Override + protected void onPause() { + super.onPause(); + stopDetector(); + } + + void startDetector() { + if (detector == null || !detector.isRunning()) { + // check permission + String permission = "android.permission.CAMERA"; + int res = getApplicationContext().checkCallingOrSelfPermission(permission); + if (res == PackageManager.PERMISSION_GRANTED) { + Log.e(LOG_TAG, "HAS PERM"); + has_camera_permission = true; + } else { + Log.e(LOG_TAG, "NO PERM"); + ActivityCompat.requestPermissions(this, + new String[]{Manifest.permission.CAMERA}, + PERMISSIONS_REQUEST_CAMERA); + } + + if(has_camera_permission) + { + if(detector == null) + { +// SurfaceView surfaceView = (SurfaceView) findViewById(R.id.surfaceView); + detector = new CameraDetector(this, CameraDetector.CameraType.CAMERA_FRONT, cameraPreview); + detector.setLicensePath("emotionhero_dev.license"); + detector.setMaxProcessRate(10); + detector.setDetectAllEmotions(true); + detector.setImageListener(this); + detector.setOnCameraEventListener(this); + } + + detector.start(); + mContentView.setText("STARTING..."); + Log.d(LOG_TAG, Boolean.toString(detector.isRunning())); + } + } + } + + void stopDetector() { + if (detector != null && detector.isRunning()) { + detector.stop(); + } + } + + private void hide() { + // Hide UI first + ActionBar actionBar = getSupportActionBar(); + if (actionBar != null) { + actionBar.hide(); + } + mControlsView.setVisibility(View.GONE); + mVisible = false; + + // Schedule a runnable to remove the status and navigation bar after a delay + mHideHandler.removeCallbacks(mShowPart2Runnable); + mHideHandler.postDelayed(mHidePart2Runnable, UI_ANIMATION_DELAY); + } + + @SuppressLint("InlinedApi") + private void show() { + // Show the system bar + mContentView.setSystemUiVisibility(View.SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN + | View.SYSTEM_UI_FLAG_LAYOUT_HIDE_NAVIGATION); + mVisible = true; + + // Schedule a runnable to display UI elements after a delay + mHideHandler.removeCallbacks(mHidePart2Runnable); + mHideHandler.postDelayed(mShowPart2Runnable, UI_ANIMATION_DELAY); + } + + /** + * Schedules a call to hide() in [delay] milliseconds, canceling any + * previously scheduled calls. + */ + private void delayedHide(int delayMillis) { + mHideHandler.removeCallbacks(mHideRunnable); + mHideHandler.postDelayed(mHideRunnable, delayMillis); + } + + @Override + /** + * Detector callback gives the faces found so we can match their scores to the given scenario. + */ + public void onImageResults(List list, Frame frame, float timestamp) { +// Log.e(LOG_TAG, "RESULT! faces: " + Integer.toString(list.size()) + " t: " + Float.toString(timestamp) + "s" ); + if(!currentScenario.isWithinTime(timestamp)) + { + mContentView.setText(String.format("LEVEL ENDED\nScore: %.2f", currentScenario.getTotalScore())); + stopDetector(); + return; + } + if (list == null) + return; + if (list.size() == 0) { + mContentView.setText("NO FACE FOUND"); + } else { + Face face = list.get(0); + currentScenario.validateFaceOnTime(face, timestamp); + mContentView.setText(String.format("SCORE \n%.2f",currentScenario.getTotalScore())); + } + } + + @Override + /** + * For CameraDetector.CameraEventListener + * Used to scale video preview output + */ + public void onCameraSizeSelected(int width, int height, Frame.ROTATE rotate) { + if (rotate == Frame.ROTATE.BY_90_CCW || rotate == Frame.ROTATE.BY_90_CW) { + previewWidth = height; + previewHeight = width; + } else { + previewHeight = height; + previewWidth = width; + } + cameraPreview.requestLayout(); + } + + @Override + public void onRequestPermissionsResult(int requestCode, + String permissions[], int[] grantResults) { + switch (requestCode) { + case PERMISSIONS_REQUEST_CAMERA: { + // If request is cancelled, the result arrays are empty. + if (grantResults.length > 0 + && grantResults[0] == PackageManager.PERMISSION_GRANTED) { + has_camera_permission = true; + startDetector(); + } else { + has_camera_permission = false; + Toast errorMsg = Toast.makeText(this, R.string.camera_required, Toast.LENGTH_LONG); + errorMsg.show(); + } + return; + } + + // other 'case' lines to check for other + // permissions this app might request + } + } + + @Override + public void onFaceDetectionStarted() + { + mContentView.setText("START!"); + } + + @Override + public void onFaceDetectionStopped() + { + mContentView.setText("No face found..."); + } +} diff --git a/app/src/main/java/com/rubenvandeven/emotionhero/Scenario.java b/app/src/main/java/com/rubenvandeven/emotionhero/Scenario.java new file mode 100644 index 0000000..c6c1a2d --- /dev/null +++ b/app/src/main/java/com/rubenvandeven/emotionhero/Scenario.java @@ -0,0 +1,136 @@ +package com.rubenvandeven.emotionhero; + +import android.util.Log; + +import com.affectiva.android.affdex.sdk.detector.Face; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +/** + * Created by ruben on 16/08/16. + */ + +abstract public class Scenario { + + float duration = 0; + + ArrayList targets = new ArrayList<>(); + + abstract void createScenario(); + + class Target { + public Emotion emotion; + public float value; + public float timestamp; + } + + ArrayList scores = new ArrayList<>(); + + class Score { + public float value; + /** + * Extra bonus given + */ + public boolean isSpotOn = false; + /** + * The target the score is awarded for + */ + public Target target; + } + + /** + * Constructor + */ + public Scenario() + { + createScenario(); + } + + /** + * Add a target on given timestamp + * @param emotion + * @param value + * @param timestamp + */ + public void setTarget(Emotion emotion, float value, float timestamp) + { + Log.e("SET", Float.toString(timestamp) + " " + Float.toString(duration)); + if(timestamp > duration) + { + duration = timestamp; + } + Target target = new Target(); + target.timestamp = timestamp; + target.value = value; + target.emotion = emotion; + targets.add(target); + } + + /** + * Add target after existing targets, give delta with last item instead of absolute time. + * @param emotion + * @param value + * @param interval + */ + public void addTarget(Emotion emotion, float value, float interval) + { + float timestamp; + if(targets.isEmpty()) { + timestamp = interval; + } else { + timestamp = targets.get(targets.size() - 1).timestamp + interval; + } + setTarget(emotion, value, timestamp); + } + + public void validateFaceOnTime(Face face, float timestamp) + { +// TODO: interpolation of time + for (int i = targets.size() - 1; i >= 0; i--) { + Target target = targets.get(i); + if(target.timestamp > timestamp - 0.2 && target.timestamp < timestamp + 0.2) { + float scored_value = target.emotion.getValueFromFace(face); + float required_value = target.value; + Score score = new Score(); + score.value = 100 - Math.abs(scored_value-required_value); + score.target = target; + scores.add(score); + } + } + } + + public float getTotalScore() + { + float value = 0; + for (Score score : scores) { + value += score.value; + } + return value; + } + + /** + * Check whether given timestamp is within duration of the scenario + * @param timestamp + * @return + */ + public boolean isWithinTime(float timestamp) + { + Log.e("TEST", Float.toString(timestamp) + " " + Float.toString(duration)); + return timestamp <= duration; + } + +} + +class ScenarioAnger extends Scenario{ + void createScenario() + { + Log.e("TESTING", "CREATE SCENARIO!!!!"); + setTarget(Emotion.ANGER, 10, 1); + setTarget(Emotion.ANGER, 20, 2); + setTarget(Emotion.ANGER, 40, 3); + setTarget(Emotion.ANGER, 70, 4); + setTarget(Emotion.ANGER, 100, 5); + } +} \ No newline at end of file diff --git a/app/src/main/res/layout/activity_gaming.xml b/app/src/main/res/layout/activity_gaming.xml new file mode 100644 index 0000000..7f866ce --- /dev/null +++ b/app/src/main/res/layout/activity_gaming.xml @@ -0,0 +1,65 @@ + + + + + + + + + + + + + + + + +