2015-06-23 22:44:24 +02:00
package com.affectiva.affdexme ;
2015-12-19 04:17:20 +01:00
import android.Manifest ;
2015-12-23 07:10:49 +01:00
import android.content.Context ;
2015-12-19 04:17:20 +01:00
import android.content.DialogInterface ;
2015-07-20 21:53:09 +02:00
import android.content.Intent ;
2015-06-23 22:44:24 +02:00
import android.content.SharedPreferences ;
import android.content.pm.PackageManager ;
import android.graphics.Typeface ;
import android.os.Bundle ;
import android.os.SystemClock ;
2015-07-20 21:53:09 +02:00
import android.preference.PreferenceManager ;
2015-12-19 04:17:20 +01:00
import android.support.annotation.NonNull ;
import android.support.v4.app.ActivityCompat ;
import android.support.v4.content.ContextCompat ;
import android.support.v7.app.AlertDialog ;
import android.support.v7.app.AppCompatActivity ;
2015-10-02 03:12:00 +02:00
import android.util.DisplayMetrics ;
2015-12-19 04:17:20 +01:00
import android.util.Log ;
2015-06-23 22:44:24 +02:00
import android.view.KeyEvent ;
import android.view.MotionEvent ;
import android.view.SurfaceView ;
import android.view.View ;
import android.view.ViewGroup ;
import android.view.WindowManager ;
2015-12-19 04:17:20 +01:00
import android.widget.Button ;
2015-07-20 21:53:09 +02:00
import android.widget.ImageButton ;
2015-06-23 22:44:24 +02:00
import android.widget.LinearLayout ;
2015-08-20 17:24:17 +02:00
import android.widget.ProgressBar ;
2015-06-23 22:44:24 +02:00
import android.widget.RelativeLayout ;
import android.widget.TextView ;
2015-08-20 17:24:17 +02:00
import android.widget.Toast ;
2015-06-23 22:44:24 +02:00
import com.affectiva.android.affdex.sdk.Frame ;
import com.affectiva.android.affdex.sdk.Frame.ROTATE ;
import com.affectiva.android.affdex.sdk.detector.CameraDetector ;
import com.affectiva.android.affdex.sdk.detector.Detector ;
import com.affectiva.android.affdex.sdk.detector.Face ;
2015-12-23 07:10:49 +01:00
import java.lang.reflect.InvocationTargetException ;
2015-12-19 04:17:20 +01:00
import java.lang.reflect.Method ;
import java.util.ArrayList ;
import java.util.List ;
2015-12-23 07:10:49 +01:00
import java.util.Locale ;
2015-12-19 04:17:20 +01:00
2015-06-23 22:44:24 +02:00
/ *
* AffdexMe is an app that demonstrates the use of the Affectiva Android SDK . It uses the
* front - facing camera on your Android device to view , process and analyze live video of your face .
* Start the app and you will see your own face on the screen and metrics describing your
* expressions .
*
* Tapping the screen will bring up a menu with options to display the Processed Frames Per Second metric ,
* display facial tracking points , and control the rate at which frames are processed by the SDK .
*
* Most of the methods in this file control the application ' s UI . Therefore , if you are just interested in learning how the Affectiva SDK works ,
2015-07-13 20:17:57 +02:00
* you will find the calls relevant to the use of the SDK in the initializeCameraDetector ( ) , startCamera ( ) , stopCamera ( ) ,
* and onImageResults ( ) methods .
2015-06-23 22:44:24 +02:00
*
* This class implements the Detector . ImageListener interface , allowing it to receive the onImageResults ( ) event .
* This class implements the Detector . FaceListener interface , allowing it to receive the onFaceDetectionStarted ( ) and
* onFaceDetectionStopped ( ) events .
2015-07-13 20:17:57 +02:00
* This class implements the CameraDetector . CameraSurfaceViewListener interface , allowing it to receive
* onSurfaceViewAspectRatioChanged ( ) events .
2015-06-23 22:44:24 +02:00
*
* In order to use this project , you will need to :
* - Obtain the SDK from Affectiva ( visit http : //www.affdex.com/mobile-sdk)
* - Copy the SDK assets folder contents into this project ' s assets folder
2015-09-24 02:08:24 +02:00
* - Copy the contents of the SDK ' s libs folder into this project ' s libs folder under AffdexMe / app / lib
* - Copy the armeabi - v7a folder ( found in the SDK libs folder ) into this project ' s jniLibs folder under AffdexMe / app / src / main / jniLibs
* - Add your license file to the / assets / Affdex folder and rename to license . txt .
* ( Note : if you name the license file something else you will need to update the licensePath in the initializeCameraDetector ( ) method in MainActivity )
2015-06-23 22:44:24 +02:00
* - Build the project
* - Run the app on an Android device with a front - facing camera
*
* Copyright ( c ) 2014 Affectiva . All rights reserved .
* /
2015-12-19 04:17:20 +01:00
public class MainActivity extends AppCompatActivity
implements Detector . FaceListener , Detector . ImageListener , CameraDetector . CameraEventListener ,
View . OnTouchListener , ActivityCompat . OnRequestPermissionsResultCallback {
2015-06-23 22:44:24 +02:00
2015-12-23 07:10:49 +01:00
public static final int MAX_SUPPORTED_FACES = 4 ;
2015-08-20 17:24:17 +02:00
public static final int NUM_METRICS_DISPLAYED = 6 ;
2015-12-19 04:17:20 +01:00
private static final String LOG_TAG = " Affectiva " ;
private static final int AFFDEXME_PERMISSIONS_REQUEST = 42 ; //value is arbitrary (between 0 and 255)
int cameraPreviewWidth = 0 ;
int cameraPreviewHeight = 0 ;
CameraDetector . CameraType cameraType ;
boolean mirrorPoints = false ;
2015-12-23 07:10:49 +01:00
private boolean cameraPermissionsAvailable = false ;
2015-06-23 22:44:24 +02:00
private CameraDetector detector = null ;
private RelativeLayout metricViewLayout ;
private LinearLayout leftMetricsLayout ;
private LinearLayout rightMetricsLayout ;
2015-08-20 17:24:17 +02:00
private MetricDisplay [ ] metricDisplays ;
private TextView [ ] metricNames ;
2015-06-23 22:44:24 +02:00
private TextView fpsName ;
private TextView fpsPct ;
2015-08-20 17:24:17 +02:00
private TextView pleaseWaitTextView ;
private ProgressBar progressBar ;
2015-06-23 22:44:24 +02:00
private RelativeLayout mainLayout ; //layout, to be resized, containing all UI elements
private RelativeLayout progressBarLayout ; //layout used to show progress circle while camera is starting
2015-12-19 04:17:20 +01:00
private LinearLayout permissionsUnavailableLayout ; //layout used to notify the user that not enough permissions have been granted to use the app
2015-06-23 22:44:24 +02:00
private SurfaceView cameraView ; //SurfaceView used to display camera images
private DrawingView drawingView ; //SurfaceView containing its own thread, used to draw facial tracking dots
2015-07-20 21:53:09 +02:00
private ImageButton settingsButton ;
2015-08-27 18:07:37 +02:00
private ImageButton cameraButton ;
2015-06-23 22:44:24 +02:00
private boolean isMenuVisible = false ;
private boolean isFPSVisible = false ;
2015-08-20 17:24:17 +02:00
private boolean isMenuShowingForFirstTime = true ;
2015-06-23 22:44:24 +02:00
private long firstSystemTime = 0 ;
private float numberOfFrames = 0 ;
private long timeToUpdate = 0 ;
private boolean isFrontFacingCameraDetected = true ;
2015-08-27 18:07:37 +02:00
private boolean isBackFacingCameraDetected = true ;
2015-12-23 07:10:49 +01:00
private boolean multiFaceModeEnabled = false ;
2015-06-23 22:44:24 +02:00
@Override
protected void onCreate ( Bundle savedInstanceState ) {
super . onCreate ( savedInstanceState ) ;
getWindow ( ) . addFlags ( WindowManager . LayoutParams . FLAG_FULLSCREEN ) ; //To maximize UI space, we declare our app to be full-screen
2015-12-23 07:10:49 +01:00
preproccessMetricImages ( ) ;
2015-06-23 22:44:24 +02:00
setContentView ( R . layout . activity_main ) ;
2015-08-20 17:24:17 +02:00
initializeUI ( ) ;
2015-12-19 04:17:20 +01:00
checkForDangerousPermissions ( ) ;
2015-08-27 18:07:37 +02:00
determineCameraAvailability ( ) ;
initializeCameraDetector ( ) ;
}
2015-12-23 07:10:49 +01:00
/ * *
* Only load the files onto disk the first time the app opens
* /
private void preproccessMetricImages ( ) {
Context context = getBaseContext ( ) ;
for ( Face . EMOJI emoji : Face . EMOJI . values ( ) ) {
if ( emoji . equals ( Face . EMOJI . UNKNOWN ) ) {
continue ;
}
String emojiResourceName = emoji . name ( ) . trim ( ) . replace ( ' ' , '_' ) . toLowerCase ( Locale . US ) . concat ( " _emoji " ) ;
String emojiFileName = emojiResourceName + " .png " ;
ImageHelper . preproccessImageIfNecessary ( context , emojiFileName , emojiResourceName ) ;
}
ImageHelper . preproccessImageIfNecessary ( context , " female_glasses.png " , " female_glasses " ) ;
ImageHelper . preproccessImageIfNecessary ( context , " female_noglasses.png " , " female_noglasses " ) ;
ImageHelper . preproccessImageIfNecessary ( context , " male_glasses.png " , " male_glasses " ) ;
ImageHelper . preproccessImageIfNecessary ( context , " male_noglasses.png " , " male_noglasses " ) ;
ImageHelper . preproccessImageIfNecessary ( context , " unknown_glasses.png " , " unknown_glasses " ) ;
ImageHelper . preproccessImageIfNecessary ( context , " unknown_noglasses.png " , " unknown_noglasses " ) ;
}
2015-12-19 04:17:20 +01:00
private void checkForDangerousPermissions ( ) {
cameraPermissionsAvailable =
ContextCompat . checkSelfPermission (
getApplicationContext ( ) ,
Manifest . permission . CAMERA ) = = PackageManager . PERMISSION_GRANTED ;
2015-12-23 07:10:49 +01:00
if ( ! cameraPermissionsAvailable ) {
2015-12-19 04:17:20 +01:00
// Should we show an explanation?
2015-12-23 07:10:49 +01:00
if ( ActivityCompat . shouldShowRequestPermissionRationale ( this , Manifest . permission . CAMERA ) ) {
2015-12-19 04:17:20 +01:00
// Show an explanation to the user *asynchronously* -- don't block
// this thread waiting for the user's response! After the user
// sees the explanation, try again to request the permission.
showPermissionExplanationDialog ( ) ;
} else {
// No explanation needed, we can request the permission.
requestNeededPermissions ( ) ;
}
}
}
private void requestNeededPermissions ( ) {
List < String > neededPermissions = new ArrayList < > ( ) ;
if ( ! cameraPermissionsAvailable ) {
neededPermissions . add ( Manifest . permission . CAMERA ) ;
}
ActivityCompat . requestPermissions (
this ,
neededPermissions . toArray ( new String [ neededPermissions . size ( ) ] ) ,
AFFDEXME_PERMISSIONS_REQUEST ) ;
// AFFDEXME_PERMISSIONS_REQUEST is an app-defined int constant that must be between 0 and 255.
// The callback method gets the result of the request.
}
@Override
public void onRequestPermissionsResult ( int requestCode , @NonNull String [ ] permissions , @NonNull int [ ] grantResults ) {
super . onRequestPermissionsResult ( requestCode , permissions , grantResults ) ;
if ( requestCode = = AFFDEXME_PERMISSIONS_REQUEST ) {
for ( int i = 0 ; i < permissions . length ; i + + ) {
String permission = permissions [ i ] ;
int grantResult = grantResults [ i ] ;
if ( permission . equals ( Manifest . permission . CAMERA ) ) {
cameraPermissionsAvailable = ( grantResult = = PackageManager . PERMISSION_GRANTED ) ;
}
}
}
2015-12-23 07:10:49 +01:00
if ( ! cameraPermissionsAvailable ) {
2015-12-19 04:17:20 +01:00
permissionsUnavailableLayout . setVisibility ( View . VISIBLE ) ;
} else {
permissionsUnavailableLayout . setVisibility ( View . GONE ) ;
}
}
private void showPermissionExplanationDialog ( ) {
final AlertDialog . Builder alertDialogBuilder = new AlertDialog . Builder (
getApplicationContext ( ) ) ;
// set title
alertDialogBuilder . setTitle ( getResources ( ) . getString ( R . string . insufficient_permissions ) ) ;
// set dialog message
alertDialogBuilder
. setMessage ( getResources ( ) . getString ( R . string . permissions_needed_explanation ) )
. setCancelable ( false )
. setPositiveButton ( getResources ( ) . getString ( R . string . understood ) , new DialogInterface . OnClickListener ( ) {
public void onClick ( DialogInterface dialog , int id ) {
dialog . cancel ( ) ;
requestNeededPermissions ( ) ;
}
} ) ;
// create alert dialog
AlertDialog alertDialog = alertDialogBuilder . create ( ) ;
// show it
alertDialog . show ( ) ;
}
2015-08-27 18:07:37 +02:00
/ * *
* We check to make sure the device has a front - facing camera .
* If it does not , we obscure the app with a notice informing the user they cannot
* use the app .
* /
void determineCameraAvailability ( ) {
PackageManager manager = getPackageManager ( ) ;
isFrontFacingCameraDetected = manager . hasSystemFeature ( PackageManager . FEATURE_CAMERA_FRONT ) ;
isBackFacingCameraDetected = manager . hasSystemFeature ( PackageManager . FEATURE_CAMERA ) ;
if ( ! isFrontFacingCameraDetected & & ! isBackFacingCameraDetected ) {
2015-08-20 17:24:17 +02:00
progressBar . setVisibility ( View . INVISIBLE ) ;
pleaseWaitTextView . setVisibility ( View . INVISIBLE ) ;
2015-06-23 22:44:24 +02:00
TextView notFoundTextView = ( TextView ) findViewById ( R . id . not_found_textview ) ;
notFoundTextView . setVisibility ( View . VISIBLE ) ;
}
2015-12-23 07:10:49 +01:00
//set default camera settings
SharedPreferences sharedPreferences = PreferenceManager . getDefaultSharedPreferences ( this ) ;
//restore the camera type settings
String cameraTypeName = sharedPreferences . getString ( " cameraType " , CameraDetector . CameraType . CAMERA_FRONT . name ( ) ) ;
if ( cameraTypeName . equals ( CameraDetector . CameraType . CAMERA_FRONT . name ( ) ) ) {
2015-08-27 18:07:37 +02:00
cameraType = CameraDetector . CameraType . CAMERA_FRONT ;
2015-08-28 21:10:43 +02:00
mirrorPoints = true ;
2015-12-23 07:10:49 +01:00
} else {
cameraType = CameraDetector . CameraType . CAMERA_BACK ;
mirrorPoints = false ;
2015-08-28 21:10:43 +02:00
}
2015-06-23 22:44:24 +02:00
}
void initializeUI ( ) {
//Get handles to UI objects
2015-12-23 07:10:49 +01:00
ViewGroup activityLayout = ( ViewGroup ) findViewById ( android . R . id . content ) ;
2015-06-23 22:44:24 +02:00
progressBarLayout = ( RelativeLayout ) findViewById ( R . id . progress_bar_cover ) ;
2015-12-19 04:17:20 +01:00
permissionsUnavailableLayout = ( LinearLayout ) findViewById ( R . id . permissionsUnavialableLayout ) ;
2015-06-23 22:44:24 +02:00
metricViewLayout = ( RelativeLayout ) findViewById ( R . id . metric_view_group ) ;
leftMetricsLayout = ( LinearLayout ) findViewById ( R . id . left_metrics ) ;
rightMetricsLayout = ( LinearLayout ) findViewById ( R . id . right_metrics ) ;
mainLayout = ( RelativeLayout ) findViewById ( R . id . main_layout ) ;
fpsPct = ( TextView ) findViewById ( R . id . fps_value ) ;
fpsName = ( TextView ) findViewById ( R . id . fps_name ) ;
cameraView = ( SurfaceView ) findViewById ( R . id . camera_preview ) ;
drawingView = ( DrawingView ) findViewById ( R . id . drawing_view ) ;
2015-07-20 21:53:09 +02:00
settingsButton = ( ImageButton ) findViewById ( R . id . settings_button ) ;
2015-08-27 18:07:37 +02:00
cameraButton = ( ImageButton ) findViewById ( R . id . camera_button ) ;
2015-08-20 17:24:17 +02:00
progressBar = ( ProgressBar ) findViewById ( R . id . progress_bar ) ;
pleaseWaitTextView = ( TextView ) findViewById ( R . id . please_wait_textview ) ;
2015-12-23 07:10:49 +01:00
Button retryPermissionsButton = ( Button ) findViewById ( R . id . retryPermissionsButton ) ;
2015-08-20 17:24:17 +02:00
//Initialize views to display metrics
metricNames = new TextView [ NUM_METRICS_DISPLAYED ] ;
metricNames [ 0 ] = ( TextView ) findViewById ( R . id . metric_name_0 ) ;
metricNames [ 1 ] = ( TextView ) findViewById ( R . id . metric_name_1 ) ;
metricNames [ 2 ] = ( TextView ) findViewById ( R . id . metric_name_2 ) ;
metricNames [ 3 ] = ( TextView ) findViewById ( R . id . metric_name_3 ) ;
metricNames [ 4 ] = ( TextView ) findViewById ( R . id . metric_name_4 ) ;
metricNames [ 5 ] = ( TextView ) findViewById ( R . id . metric_name_5 ) ;
metricDisplays = new MetricDisplay [ NUM_METRICS_DISPLAYED ] ;
metricDisplays [ 0 ] = ( MetricDisplay ) findViewById ( R . id . metric_pct_0 ) ;
metricDisplays [ 1 ] = ( MetricDisplay ) findViewById ( R . id . metric_pct_1 ) ;
metricDisplays [ 2 ] = ( MetricDisplay ) findViewById ( R . id . metric_pct_2 ) ;
metricDisplays [ 3 ] = ( MetricDisplay ) findViewById ( R . id . metric_pct_3 ) ;
metricDisplays [ 4 ] = ( MetricDisplay ) findViewById ( R . id . metric_pct_4 ) ;
metricDisplays [ 5 ] = ( MetricDisplay ) findViewById ( R . id . metric_pct_5 ) ;
2015-06-23 22:44:24 +02:00
//Load Application Font and set UI Elements to use it
Typeface face = Typeface . createFromAsset ( getAssets ( ) , " fonts/Square.ttf " ) ;
2015-08-20 17:24:17 +02:00
for ( TextView textView : metricNames ) {
textView . setTypeface ( face ) ;
}
for ( MetricDisplay metricDisplay : metricDisplays ) {
metricDisplay . setTypeface ( face ) ;
}
2015-06-23 22:44:24 +02:00
fpsPct . setTypeface ( face ) ;
fpsName . setTypeface ( face ) ;
2015-07-20 21:53:09 +02:00
drawingView . setTypeface ( face ) ;
2015-08-20 17:24:17 +02:00
pleaseWaitTextView . setTypeface ( face ) ;
2015-06-23 22:44:24 +02:00
//Hide left and right metrics by default (will be made visible when face detection starts)
leftMetricsLayout . setAlpha ( 0 ) ;
rightMetricsLayout . setAlpha ( 0 ) ;
/ * *
* This app uses two SurfaceView objects : one to display the camera image and the other to draw facial tracking dots .
* Since we want the tracking dots to appear over the camera image , we use SurfaceView . setZOrderMediaOverlay ( ) to indicate that
* cameraView represents our ' media ' , and drawingView represents our ' overlay ' , so that Android will render them in the
* correct order .
* /
drawingView . setZOrderMediaOverlay ( true ) ;
cameraView . setZOrderMediaOverlay ( false ) ;
//Attach event listeners to the menu and edit box
activityLayout . setOnTouchListener ( this ) ;
2015-07-13 20:17:57 +02:00
/ *
2015-06-23 22:44:24 +02:00
* This app sets the View . SYSTEM_UI_FLAG_HIDE_NAVIGATION flag . Unfortunately , this flag causes
* Android to steal the first touch event after the navigation bar has been hidden , a touch event
* which should be used to make our menu visible again . Therefore , we attach a listener to be notified
* when the navigation bar has been made visible again , which corresponds to the touch event that Android
* steals . If the menu bar was not visible , we make it visible .
* /
activityLayout . setOnSystemUiVisibilityChangeListener ( new View . OnSystemUiVisibilityChangeListener ( ) {
@Override
public void onSystemUiVisibilityChange ( int uiCode ) {
2015-12-23 07:10:49 +01:00
if ( ( uiCode = = 0 ) & & ( ! isMenuVisible ) ) {
2015-06-23 22:44:24 +02:00
setMenuVisible ( true ) ;
}
}
} ) ;
2015-12-19 04:17:20 +01:00
retryPermissionsButton . setOnClickListener ( new View . OnClickListener ( ) {
@Override
public void onClick ( View v ) {
requestNeededPermissions ( ) ;
}
} ) ;
2015-06-23 22:44:24 +02:00
}
2015-07-13 20:17:57 +02:00
void initializeCameraDetector ( ) {
/ * Put the SDK in camera mode by using this constructor . The SDK will be in control of
* the camera . If a SurfaceView is passed in as the last argument to the constructor ,
* that view will be painted with what the camera sees .
* /
2015-12-23 07:10:49 +01:00
detector = new CameraDetector ( this , cameraType , cameraView , ( multiFaceModeEnabled ? MAX_SUPPORTED_FACES : 1 ) , Detector . FaceDetectorMode . LARGE_FACES ) ;
2015-09-24 02:08:24 +02:00
2015-12-19 04:17:20 +01:00
// update the license path here if you name your file something else
detector . setLicensePath ( " license.txt " ) ;
2015-07-13 20:17:57 +02:00
detector . setImageListener ( this ) ;
detector . setFaceListener ( this ) ;
2015-08-27 18:07:37 +02:00
detector . setOnCameraEventListener ( this ) ;
2015-07-13 20:17:57 +02:00
}
/ *
* We use onResume ( ) to restore application settings using the SharedPreferences object
2015-06-23 22:44:24 +02:00
* /
@Override
public void onResume ( ) {
super . onResume ( ) ;
2015-12-19 04:17:20 +01:00
checkForDangerousPermissions ( ) ;
2015-06-23 22:44:24 +02:00
restoreApplicationSettings ( ) ;
2015-08-20 17:24:17 +02:00
setMenuVisible ( true ) ;
isMenuShowingForFirstTime = true ;
2015-06-23 22:44:24 +02:00
}
2015-12-23 07:10:49 +01:00
private void setMultiFaceModeEnabled ( boolean isEnabled ) {
//if setting change is necessary
if ( isEnabled ! = multiFaceModeEnabled ) {
// change the setting, stop the detector, and reinitialize it to change the setting
multiFaceModeEnabled = isEnabled ;
stopDetector ( ) ;
initializeCameraDetector ( ) ;
}
}
2015-07-13 20:17:57 +02:00
/ *
2015-06-23 22:44:24 +02:00
* We use the Shared Preferences object to restore application settings .
2015-07-13 20:17:57 +02:00
* /
2015-06-23 22:44:24 +02:00
public void restoreApplicationSettings ( ) {
2015-08-20 17:24:17 +02:00
SharedPreferences sharedPreferences = PreferenceManager . getDefaultSharedPreferences ( this ) ;
2015-07-20 21:53:09 +02:00
2015-12-23 07:10:49 +01:00
//restore the camera type settings
String cameraTypeName = sharedPreferences . getString ( " cameraType " , CameraDetector . CameraType . CAMERA_FRONT . name ( ) ) ;
if ( cameraTypeName . equals ( CameraDetector . CameraType . CAMERA_FRONT . name ( ) ) ) {
setCameraType ( CameraDetector . CameraType . CAMERA_FRONT ) ;
} else {
setCameraType ( CameraDetector . CameraType . CAMERA_BACK ) ;
}
//restore the multiface mode setting to reset the detector if necessary
if ( sharedPreferences . getBoolean ( " multiface " , false ) ) { // default to false
setMultiFaceModeEnabled ( true ) ;
} else {
setMultiFaceModeEnabled ( false ) ;
}
2015-08-20 17:24:17 +02:00
//restore camera processing rate
2015-12-23 07:10:49 +01:00
int detectorProcessRate = PreferencesUtils . getFrameProcessingRate ( sharedPreferences ) ;
2015-07-20 21:53:09 +02:00
detector . setMaxProcessRate ( detectorProcessRate ) ;
2015-12-23 07:10:49 +01:00
drawingView . invalidateDimensions ( ) ;
2015-06-23 22:44:24 +02:00
2015-12-19 04:17:20 +01:00
if ( sharedPreferences . getBoolean ( " fps " , isFPSVisible ) ) { //restore isFPSMetricVisible
2015-06-23 22:44:24 +02:00
setFPSVisible ( true ) ;
} else {
setFPSVisible ( false ) ;
}
2015-12-19 04:17:20 +01:00
if ( sharedPreferences . getBoolean ( " track " , drawingView . getDrawPointsEnabled ( ) ) ) { //restore isTrackingDotsVisible
2015-06-23 22:44:24 +02:00
setTrackPoints ( true ) ;
} else {
setTrackPoints ( false ) ;
2015-07-20 21:53:09 +02:00
}
2015-12-23 07:10:49 +01:00
if ( sharedPreferences . getBoolean ( " appearance " , drawingView . getDrawAppearanceMarkersEnabled ( ) ) ) {
detector . setDetectAllAppearance ( true ) ;
setShowAppearance ( true ) ;
} else {
detector . setDetectAllAppearance ( false ) ;
setShowAppearance ( false ) ;
}
if ( sharedPreferences . getBoolean ( " emoji " , drawingView . getDrawEmojiMarkersEnabled ( ) ) ) {
detector . setDetectAllEmojis ( true ) ;
setShowEmoji ( true ) ;
2015-07-20 21:53:09 +02:00
} else {
2015-12-23 07:10:49 +01:00
detector . setDetectAllEmojis ( false ) ;
setShowEmoji ( false ) ;
2015-07-20 21:53:09 +02:00
}
2015-08-20 17:24:17 +02:00
//populate metric displays
for ( int n = 0 ; n < NUM_METRICS_DISPLAYED ; n + + ) {
2015-12-19 04:17:20 +01:00
activateMetric ( n , PreferencesUtils . getMetricFromPrefs ( sharedPreferences , n ) ) ;
2015-07-20 21:53:09 +02:00
}
2015-12-23 07:10:49 +01:00
//if we are in multiface mode, we need to enable the detection of all emotions
if ( multiFaceModeEnabled ) {
detector . setDetectAllEmotions ( true ) ;
}
2015-07-20 21:53:09 +02:00
}
2015-08-20 17:24:17 +02:00
/ * *
* Populates a TextView to display a metric name and readies a MetricDisplay to display the value .
* Uses reflection to :
2015-12-19 04:17:20 +01:00
* - enable the corresponding metric in the Detector object by calling Detector . setDetect < MetricName > ( )
* - save the Method object that will be invoked on the Face object received in onImageResults ( ) to get the metric score
2015-08-20 17:24:17 +02:00
* /
void activateMetric ( int index , MetricsManager . Metrics metric ) {
2015-07-20 21:53:09 +02:00
2015-08-20 17:24:17 +02:00
Method getFaceScoreMethod = null ; //The method that will be used to get a metric score
2015-07-20 21:53:09 +02:00
2015-12-23 07:10:49 +01:00
try {
switch ( metric . getType ( ) ) {
case Emotion :
Detector . class . getMethod ( " setDetect " + MetricsManager . getCamelCase ( metric ) , boolean . class ) . invoke ( detector , true ) ;
metricNames [ index ] . setText ( MetricsManager . getUpperCaseName ( metric ) ) ;
getFaceScoreMethod = Face . Emotions . class . getMethod ( " get " + MetricsManager . getCamelCase ( metric ) ) ;
//The MetricDisplay for Valence is unique; it shades it color depending on the metric value
if ( metric = = MetricsManager . Emotions . VALENCE ) {
metricDisplays [ index ] . setIsShadedMetricView ( true ) ;
} else {
metricDisplays [ index ] . setIsShadedMetricView ( false ) ;
}
break ;
case Expression :
Detector . class . getMethod ( " setDetect " + MetricsManager . getCamelCase ( metric ) , boolean . class ) . invoke ( detector , true ) ;
metricNames [ index ] . setText ( MetricsManager . getUpperCaseName ( metric ) ) ;
getFaceScoreMethod = Face . Expressions . class . getMethod ( " get " + MetricsManager . getCamelCase ( metric ) ) ;
break ;
case Emoji :
detector . setDetectAllEmojis ( true ) ;
MetricsManager . Emojis emoji = ( ( MetricsManager . Emojis ) metric ) ;
String metricTitle = emoji . getDisplayName ( ) ; // + " " + emoji.getUnicodeForEmoji();
metricNames [ index ] . setText ( metricTitle ) ;
Log . d ( LOG_TAG , " Getter Method: " + " get " + MetricsManager . getCamelCase ( metric ) ) ;
getFaceScoreMethod = Face . Emojis . class . getMethod ( " get " + MetricsManager . getCamelCase ( metric ) ) ;
break ;
2015-08-20 17:24:17 +02:00
}
2015-12-23 07:10:49 +01:00
} catch ( NoSuchMethodException e ) {
Log . e ( LOG_TAG , String . format ( " No such method while using reflection to generate methods for %s " , metric . toString ( ) ) , e ) ;
} catch ( InvocationTargetException e ) {
Log . e ( LOG_TAG , String . format ( " Invocation error while using reflection to generate methods for %s " , metric . toString ( ) ) , e ) ;
} catch ( IllegalAccessException e ) {
Log . e ( LOG_TAG , String . format ( " Illegal access error while using reflection to generate methods for %s " , metric . toString ( ) ) , e ) ;
2015-06-23 22:44:24 +02:00
}
2015-08-20 17:24:17 +02:00
metricDisplays [ index ] . setMetricToDisplay ( metric , getFaceScoreMethod ) ;
2015-06-23 22:44:24 +02:00
}
/ * *
* Reset the variables used to calculate processed frames per second .
2015-12-19 04:17:20 +01:00
* * /
2015-06-23 22:44:24 +02:00
public void resetFPSCalculations ( ) {
firstSystemTime = SystemClock . elapsedRealtime ( ) ;
timeToUpdate = firstSystemTime + 1000L ;
numberOfFrames = 0 ;
}
/ * *
2015-09-10 19:30:10 +02:00
* We want to start the camera as late as possible , so it does not freeze the application before it has been visually resumed .
* We thus post a runnable that will take care of starting the camera .
2015-06-23 22:44:24 +02:00
* We also reset variables used to calculate the Processed Frames Per Second .
* /
@Override
public void onWindowFocusChanged ( boolean hasFocus ) {
if ( hasFocus & & isFrontFacingCameraDetected ) {
2015-09-10 19:30:10 +02:00
cameraView . post ( new Runnable ( ) {
@Override
public void run ( ) {
mainWindowResumedTasks ( ) ;
}
} ) ;
}
}
2015-07-13 20:17:57 +02:00
2015-08-20 17:24:17 +02:00
void mainWindowResumedTasks ( ) {
2015-08-27 18:07:37 +02:00
2015-12-19 04:17:20 +01:00
//Notify the user that they can't use the app without authorizing these permissions.
2015-12-23 07:10:49 +01:00
if ( ! cameraPermissionsAvailable ) {
2015-12-19 04:17:20 +01:00
permissionsUnavailableLayout . setVisibility ( View . VISIBLE ) ;
return ;
}
2015-08-27 18:07:37 +02:00
startDetector ( ) ;
if ( ! drawingView . isDimensionsNeeded ( ) ) {
2015-08-20 17:24:17 +02:00
progressBarLayout . setVisibility ( View . GONE ) ;
2015-06-23 22:44:24 +02:00
}
2015-08-20 17:24:17 +02:00
resetFPSCalculations ( ) ;
cameraView . postDelayed ( new Runnable ( ) {
@Override
public void run ( ) {
if ( isMenuShowingForFirstTime ) {
setMenuVisible ( false ) ;
}
}
2015-08-27 18:07:37 +02:00
} , 5000 ) ;
2015-06-23 22:44:24 +02:00
}
2015-08-27 18:07:37 +02:00
void startDetector ( ) {
if ( ! isBackFacingCameraDetected & & ! isFrontFacingCameraDetected )
return ; //without any cameras detected, we cannot proceed
2015-08-20 17:24:17 +02:00
2015-08-27 18:07:37 +02:00
detector . setDetectValence ( true ) ; //this app will always detect valence
2015-07-13 20:17:57 +02:00
if ( ! detector . isRunning ( ) ) {
2015-06-23 22:44:24 +02:00
try {
detector . start ( ) ;
} catch ( Exception e ) {
Log . e ( LOG_TAG , e . getMessage ( ) ) ;
}
}
2015-07-13 20:17:57 +02:00
}
2015-06-23 22:44:24 +02:00
2015-08-27 18:07:37 +02:00
2015-06-23 22:44:24 +02:00
@Override
public void onFaceDetectionStarted ( ) {
leftMetricsLayout . animate ( ) . alpha ( 1 ) ; //make left and right metrics appear
rightMetricsLayout . animate ( ) . alpha ( 1 ) ;
2015-07-20 21:53:09 +02:00
2015-06-23 22:44:24 +02:00
resetFPSCalculations ( ) ; //Since the FPS may be different whether a face is being tracked or not, reset variables.
}
@Override
public void onFaceDetectionStopped ( ) {
performFaceDetectionStoppedTasks ( ) ;
}
void performFaceDetectionStoppedTasks ( ) {
leftMetricsLayout . animate ( ) . alpha ( 0 ) ; //make left and right metrics disappear
rightMetricsLayout . animate ( ) . alpha ( 0 ) ;
resetFPSCalculations ( ) ; //Since the FPS may be different whether a face is being tracked or not, reset variables.
}
/ * *
* This event is received every time the SDK processes a frame .
* /
@Override
public void onImageResults ( List < Face > faces , Frame image , float timeStamp ) {
//If the faces object is null, we received an unprocessed frame
if ( faces = = null ) {
return ;
}
//At this point, we know the frame received was processed, so we perform our processed frames per second calculations
performFPSCalculations ( ) ;
//If faces.size() is 0, we received a frame in which no face was detected
2015-12-23 07:10:49 +01:00
if ( faces . size ( ) < = 0 ) {
drawingView . invalidatePoints ( ) ;
} else if ( faces . size ( ) = = 1 ) {
metricViewLayout . setVisibility ( View . VISIBLE ) ;
//update metrics with latest face information. The metrics are displayed on a MetricView, a custom view with a .setScore() method.
for ( MetricDisplay metricDisplay : metricDisplays ) {
updateMetricScore ( metricDisplay , faces . get ( 0 ) ) ;
}
2015-06-23 22:44:24 +02:00
2015-12-23 07:10:49 +01:00
/ * *
* If the user has selected to have any facial attributes drawn , we use face . getFacePoints ( ) to send those points
* to our drawing thread and also inform the thread what the valence score was , as that will determine the color
* of the bounding box .
* /
if ( drawingView . getDrawPointsEnabled ( ) | | drawingView . getDrawAppearanceMarkersEnabled ( ) | | drawingView . getDrawEmojiMarkersEnabled ( ) ) {
drawingView . updatePoints ( faces , mirrorPoints ) ;
}
2015-06-23 22:44:24 +02:00
2015-12-23 07:10:49 +01:00
} else {
// metrics overlay is hidden in multi face mode
metricViewLayout . setVisibility ( View . GONE ) ;
2015-06-23 22:44:24 +02:00
2015-12-23 07:10:49 +01:00
// always update points in multi face mode
drawingView . updatePoints ( faces , mirrorPoints ) ;
2015-06-23 22:44:24 +02:00
}
}
2015-08-20 17:24:17 +02:00
/ * *
* Use the method that we saved in activateMetric ( ) to get the metric score and display it
* /
void updateMetricScore ( MetricDisplay metricDisplay , Face face ) {
MetricsManager . Metrics metric = metricDisplay . getMetricToDisplay ( ) ;
float score = Float . NaN ;
try {
2015-12-23 07:10:49 +01:00
switch ( metric . getType ( ) ) {
case Emotion :
score = ( Float ) metricDisplay . getFaceScoreMethod ( ) . invoke ( face . emotions ) ;
break ;
case Expression :
score = ( Float ) metricDisplay . getFaceScoreMethod ( ) . invoke ( face . expressions ) ;
break ;
case Emoji :
score = ( Float ) metricDisplay . getFaceScoreMethod ( ) . invoke ( face . emojis ) ;
break ;
default :
throw new Exception ( " Unknown Metric Type: " + metric . getType ( ) . toString ( ) ) ;
2015-08-20 17:24:17 +02:00
}
} catch ( Exception e ) {
2015-12-19 04:17:20 +01:00
Log . e ( LOG_TAG , String . format ( " Error using reflecting to get %s score from face. " , metric . toString ( ) ) ) ;
2015-07-20 21:53:09 +02:00
}
2015-08-20 17:24:17 +02:00
metricDisplay . setScore ( score ) ;
2015-07-20 21:53:09 +02:00
}
2015-06-23 22:44:24 +02:00
/ * *
* FPS measurement simply uses SystemClock to measure how many frames were processed since
* the FPS variables were last reset .
* The constants 1000L and 1000f appear because . elapsedRealtime ( ) measures time in milliseconds .
* Note that if 20 frames per second are processed , this method could run for 1 . 5 years without being reset
* before numberOfFrames overflows .
* /
void performFPSCalculations ( ) {
numberOfFrames + = 1 ;
long currentTime = SystemClock . elapsedRealtime ( ) ;
if ( currentTime > timeToUpdate ) {
2015-12-19 04:17:20 +01:00
float framesPerSecond = ( numberOfFrames / ( float ) ( currentTime - firstSystemTime ) ) * 1000f ;
fpsPct . setText ( String . format ( " %.1f " , framesPerSecond ) ) ;
2015-06-23 22:44:24 +02:00
timeToUpdate = currentTime + 1000L ;
}
}
/ * *
* Although we start the camera in onWindowFocusChanged ( ) , we stop it in onPause ( ) , and set detector to be null so that when onWindowFocusChanged ( )
* is called it restarts the camera . We also set the Progress Bar to be visible , so the camera ( which may need resizing when the app
* is resumed ) is obscured .
* /
@Override
public void onPause ( ) {
super . onPause ( ) ;
progressBarLayout . setVisibility ( View . VISIBLE ) ;
performFaceDetectionStoppedTasks ( ) ;
2015-08-20 17:24:17 +02:00
2015-08-27 18:07:37 +02:00
stopDetector ( ) ;
}
2015-08-20 17:24:17 +02:00
2015-08-27 18:07:37 +02:00
void stopDetector ( ) {
if ( detector . isRunning ( ) ) {
try {
detector . stop ( ) ;
} catch ( Exception e ) {
2015-12-19 04:17:20 +01:00
Log . e ( LOG_TAG , e . getMessage ( ) ) ;
2015-08-27 18:07:37 +02:00
}
2015-06-23 22:44:24 +02:00
}
2015-08-27 18:07:37 +02:00
detector . setDetectAllEmotions ( false ) ;
detector . setDetectAllExpressions ( false ) ;
2015-12-23 07:10:49 +01:00
detector . setDetectAllAppearance ( false ) ;
detector . setDetectAllEmojis ( false ) ;
2015-06-23 22:44:24 +02:00
}
/ * *
* When the user taps the screen , hide the menu if it is visible and show it if it is hidden .
2015-12-19 04:17:20 +01:00
* * /
void setMenuVisible ( boolean b ) {
2015-08-20 17:24:17 +02:00
isMenuShowingForFirstTime = false ;
2015-06-23 22:44:24 +02:00
isMenuVisible = b ;
if ( b ) {
2015-07-20 21:53:09 +02:00
settingsButton . setVisibility ( View . VISIBLE ) ;
2015-08-27 18:07:37 +02:00
cameraButton . setVisibility ( View . VISIBLE ) ;
2015-06-23 22:44:24 +02:00
//We display the navigation bar again
getWindow ( ) . getDecorView ( ) . setSystemUiVisibility (
View . SYSTEM_UI_FLAG_LAYOUT_STABLE
| View . SYSTEM_UI_FLAG_LAYOUT_HIDE_NAVIGATION
| View . SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN ) ;
2015-12-19 04:17:20 +01:00
} else {
2015-06-23 22:44:24 +02:00
//We hide the navigation bar
getWindow ( ) . getDecorView ( ) . setSystemUiVisibility (
View . SYSTEM_UI_FLAG_LAYOUT_STABLE
| View . SYSTEM_UI_FLAG_LAYOUT_HIDE_NAVIGATION
| View . SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN
| View . SYSTEM_UI_FLAG_HIDE_NAVIGATION
2015-12-23 07:10:49 +01:00
| View . SYSTEM_UI_FLAG_FULLSCREEN ) ;
2015-07-20 21:53:09 +02:00
settingsButton . setVisibility ( View . INVISIBLE ) ;
2015-08-27 18:07:37 +02:00
cameraButton . setVisibility ( View . INVISIBLE ) ;
2015-06-23 22:44:24 +02:00
}
}
/ * *
* If a user has a phone with a physical menu button , they may expect it to toggle
* the menu , so we add that functionality .
* /
@Override
public boolean onKeyDown ( int keyCode , KeyEvent event ) {
if ( keyCode = = KeyEvent . KEYCODE_MENU ) {
setMenuVisible ( ! isMenuVisible ) ;
return true ;
}
return super . onKeyDown ( keyCode , event ) ;
}
//If the user selects to have facial tracking dots drawn, inform our drawing thread.
void setTrackPoints ( boolean b ) {
drawingView . setDrawPointsEnabled ( b ) ;
}
2015-12-23 07:10:49 +01:00
void setShowAppearance ( boolean b ) {
drawingView . setDrawAppearanceMarkersEnabled ( b ) ;
}
void setShowEmoji ( boolean b ) {
drawingView . setDrawEmojiMarkersEnabled ( b ) ;
2015-07-20 21:53:09 +02:00
}
2015-12-23 07:10:49 +01:00
2015-06-23 22:44:24 +02:00
void setFPSVisible ( boolean b ) {
isFPSVisible = b ;
if ( b ) {
fpsName . setVisibility ( View . VISIBLE ) ;
fpsPct . setVisibility ( View . VISIBLE ) ;
} else {
fpsName . setVisibility ( View . INVISIBLE ) ;
fpsPct . setVisibility ( View . INVISIBLE ) ;
}
}
@Override
public boolean onTouch ( View v , MotionEvent event ) {
if ( event . getAction ( ) = = MotionEvent . ACTION_DOWN ) {
setMenuVisible ( ! isMenuVisible ) ;
}
return false ;
}
2015-07-20 21:53:09 +02:00
public void settings_button_click ( View view ) {
2015-08-28 21:10:43 +02:00
startActivity ( new Intent ( this , SettingsActivity . class ) ) ;
2015-07-20 21:53:09 +02:00
}
2015-08-27 18:07:37 +02:00
2015-12-23 07:10:49 +01:00
@SuppressWarnings ( " SuspiciousNameCombination " )
2015-08-27 18:07:37 +02:00
@Override
public void onCameraSizeSelected ( int cameraWidth , int cameraHeight , ROTATE rotation ) {
if ( rotation = = ROTATE . BY_90_CCW | | rotation = = ROTATE . BY_90_CW ) {
cameraPreviewWidth = cameraHeight ;
cameraPreviewHeight = cameraWidth ;
} else {
cameraPreviewWidth = cameraWidth ;
cameraPreviewHeight = cameraHeight ;
}
2015-12-19 04:17:20 +01:00
drawingView . setThickness ( ( int ) ( cameraPreviewWidth / 100f ) ) ;
2015-08-27 18:07:37 +02:00
mainLayout . post ( new Runnable ( ) {
@Override
public void run ( ) {
2015-10-02 03:12:00 +02:00
//Get the screen width and height, and calculate the new app width/height based on the surfaceview aspect ratio.
DisplayMetrics displaymetrics = new DisplayMetrics ( ) ;
getWindowManager ( ) . getDefaultDisplay ( ) . getMetrics ( displaymetrics ) ;
int layoutWidth = displaymetrics . widthPixels ;
int layoutHeight = displaymetrics . heightPixels ;
2015-08-27 18:07:37 +02:00
if ( cameraPreviewWidth = = 0 | | cameraPreviewHeight = = 0 | | layoutWidth = = 0 | | layoutHeight = = 0 )
return ;
2015-12-19 04:17:20 +01:00
float layoutAspectRatio = ( float ) layoutWidth / layoutHeight ;
float cameraPreviewAspectRatio = ( float ) cameraPreviewWidth / cameraPreviewHeight ;
2015-08-27 18:07:37 +02:00
int newWidth ;
int newHeight ;
if ( cameraPreviewAspectRatio > layoutAspectRatio ) {
newWidth = layoutWidth ;
2015-12-19 04:17:20 +01:00
newHeight = ( int ) ( layoutWidth / cameraPreviewAspectRatio ) ;
2015-08-27 18:07:37 +02:00
} else {
newWidth = ( int ) ( layoutHeight * cameraPreviewAspectRatio ) ;
newHeight = layoutHeight ;
}
2015-12-19 04:17:20 +01:00
drawingView . updateViewDimensions ( newWidth , newHeight , cameraPreviewWidth , cameraPreviewHeight ) ;
2015-08-27 18:07:37 +02:00
ViewGroup . LayoutParams params = mainLayout . getLayoutParams ( ) ;
params . height = newHeight ;
params . width = newWidth ;
mainLayout . setLayoutParams ( params ) ;
//Now that our main layout has been resized, we can remove the progress bar that was obscuring the screen (its purpose was to obscure the resizing of the SurfaceView)
progressBarLayout . setVisibility ( View . GONE ) ;
}
} ) ;
}
public void camera_button_click ( View view ) {
2015-12-23 07:10:49 +01:00
//Toggle the camera setting
setCameraType ( cameraType = = CameraDetector . CameraType . CAMERA_FRONT ? CameraDetector . CameraType . CAMERA_BACK : CameraDetector . CameraType . CAMERA_FRONT ) ;
}
private void setCameraType ( CameraDetector . CameraType type ) {
SharedPreferences . Editor preferencesEditor = PreferenceManager . getDefaultSharedPreferences ( this ) . edit ( ) ;
//If a settings change is necessary
if ( cameraType ! = type ) {
switch ( type ) {
case CAMERA_BACK :
if ( isBackFacingCameraDetected ) {
cameraType = CameraDetector . CameraType . CAMERA_BACK ;
mirrorPoints = false ;
} else {
Toast . makeText ( this , " No back-facing camera found " , Toast . LENGTH_LONG ) . show ( ) ;
return ;
}
break ;
case CAMERA_FRONT :
if ( isFrontFacingCameraDetected ) {
cameraType = CameraDetector . CameraType . CAMERA_FRONT ;
mirrorPoints = true ;
} else {
Toast . makeText ( this , " No front-facing camera found " , Toast . LENGTH_LONG ) . show ( ) ;
return ;
}
break ;
default :
Log . e ( LOG_TAG , " Unknown camera type selected " ) ;
2015-08-27 18:07:37 +02:00
}
2015-12-23 07:10:49 +01:00
performFaceDetectionStoppedTasks ( ) ;
2015-08-27 18:07:37 +02:00
detector . setCameraType ( cameraType ) ;
2015-12-23 07:10:49 +01:00
preferencesEditor . putString ( " cameraType " , cameraType . name ( ) ) ;
preferencesEditor . apply ( ) ;
2015-08-27 18:07:37 +02:00
}
}
2015-06-23 22:44:24 +02:00
}