introduction to voice i/o on android

Bringing Your App Into The Conversation An Introduction to Voice I/O on Android

February, 2016

2

Software Engineer at IBM Mobile Innovation Lab @jorgelpaez19

Jorge Paez

3

Why?

5

Good Examples

6

Good Examples

5 Million 50 Million 100 Million

7

Demo

8

Voice Inputpublic class MainActivity extends Activity {

// More Java

@Overrideprotected void onActivityResult(int requestCode, int resultCode, Intent data) { super.onActivityResult(requestCode, resultCode, data); ears.processSound(requestCode, resultCode, data); }

}

public class Ears {

// More Java

public void startListening(Activity activity) { Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH); intent.putExtra(RecognizerIntent.EXTRA_PROMPT, promptMsg); try { activity.startActivityForResult(intent, code); } catch (ActivityNotFoundException a) { Toast.makeText(activity, noVoiceSupportMsg, Toast.LENGTH_SHORT).show(); } }

public void processSound(int requestCode, int resultCode, Intent data) { if(code == requestCode && resultCode == Activity.RESULT_OK && null != data) { ArrayList<String> strings = data.getStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS); float[] confidences = data.getFloatArrayExtra(RecognizerIntent.EXTRA_CONFIDENCE_SCORES); Guess[] guesses = new Guess[strings.size()]; for(int i = 0; i < guesses.length; ++i) { guesses[i] = new Guess(strings.get(i), confidences[i]); } notifyListeners(guesses); } }

}

9

Voice Inputpublic class

// More Java

@Overrideprotected void super ears.processSound(requestCode}

}

public class

// More Java

public void Intent intent = intent.putExtra(RecognizerIntent. try { activity.startActivityForResult(intent } catch Toast. } }

public void if(code ArrayList<String> strings = data.getStringArrayListExtra(RecognizerIntent. float Guess[] guesses = for guesses[i] = } notifyListeners(guesses) } }

}

public void startListening(Activity activity) { Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH); intent.putExtra(RecognizerIntent.EXTRA_PROMPT, promptMsg); try { activity.startActivityForResult(intent, code); } catch (ActivityNotFoundException a) { Toast.makeText(activity, noVoiceSupportMsg, Toast.LENGTH_SHORT).show(); } }

10


// More Java


}

public class

// More Java



}

@Overrideprotected void onActivityResult(int requestCode, int resultCode, Intent data) { super.onActivityResult(requestCode, resultCode, data); ears.processSound(requestCode, resultCode, data); }

11


// More Java


}

public class

// More Java



}

public void processSound(int requestCode, int resultCode, Intent data) { if(code == requestCode && resultCode == Activity.RESULT_OK && null != data) { ArrayList<String> strings = data.getStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS); float[] confidences = data.getFloatArrayExtra(RecognizerIntent.EXTRA_CONFIDENCE_SCORES); Guess[] guesses = new Guess[strings.size()]; for(int i = 0; i < guesses.length; ++i) { guesses[i] = new Guess(strings.get(i), confidences[i]); } notifyListeners(guesses); } }

12

Voice Outputpublic class Voice implements TextToSpeech.OnInitListener {

// More Java

public Voice(Context context) { this.context = context;

this.tts = new TextToSpeech(this.context, this); this.isReadyTotalk = false;

}

@Overridepublic void onInit(int status) { if (status == TextToSpeech.SUCCESS) {

isReadyTotalk = true; Log.e(TAG, “Ready to speak!”); } else { Log.e(TAG, "TTS Initialization Failed! Status: " + status); } }

public void say(String phrase, boolean shouldSayNow) { if (isReadyTotalk) { Log.e(TAG, "I'm not ready to say anything :("); } else { int queueMode = shouldSayNow ? TextToSpeech.QUEUE_FLUSH : TextToSpeech.QUEUE_ADD; String reqId = String.valueOf(new Random().nextLong()); tts.speak(phrase, queueMode, null, reqId); } }

public void onDestroy() { if (tts != null) { tts.stop(); tts.shutdown(); } context = null; }

}

13

Voice Outputpublic class

// More Java

public Voice(Context context) { this.context

this.ttsthis.isReadyTotalk

}

@Overridepublic void if (status == TextToSpeech.

isReadyTotalk Log. } else } }

public void if ( Log. } else } }

public void if ( } context }

}

implements TextToSpeech.OnInitListener {

this.tts = new TextToSpeech(this.context, this); this.isReadyTotalk = false;

@Overridepublic void onInit(int status) { if (status == TextToSpeech.SUCCESS) {

isReadyTotalk = true; Log.e(TAG, “Ready to speak!”); } else { Log.e(TAG, "TTS Initialization Failed! Status: " + status); } }

14


// More Java



}





}

public void say(String phrase, boolean shouldSayNow) { if (isReadyTotalk) { Log.e(TAG, "I'm not ready to say anything :("); } else { int queueMode = shouldSayNow ? TextToSpeech.QUEUE_FLUSH : TextToSpeech.QUEUE_ADD; String reqId = String.valueOf(new Random().nextLong()); tts.speak(phrase, queueMode, null, reqId); } }

15


// More Java



}





}

public void onDestroy() { if (tts != null) { tts.stop(); tts.shutdown(); } context = null; }

16

Demo

17

Voice Inputpublic class Ears {

private Locale locale;

// More Java

public void startListening(Activity activity) { Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH); intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM); intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE, locale); intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_PREFERENCE, locale); intent.putExtra(RecognizerIntent.EXTRA_ONLY_RETURN_LANGUAGE_PREFERENCE, locale); intent.putExtra(RecognizerIntent.EXTRA_PROMPT, promptMsg); try { activity.startActivityForResult(intent, code); } catch (ActivityNotFoundException a) { Toast.makeText(activity, noVoiceSupportMsg, Toast.LENGTH_SHORT).show(); } }

public void setToEnglish() { locale = Locale.US; } public void setToSpanish() { locale = new Locale("es", "ES"); }

}

18

Voice Language Inputpublic class

private Locale

// More Java

public void Intent intent = intent.putExtra(RecognizerIntent. intent.putExtra(RecognizerIntent. intent.putExtra(RecognizerIntent. intent.putExtra(RecognizerIntent. intent.putExtra(RecognizerIntent. try { activity.startActivityForResult(intent } catch Toast. } }

public void locale } public void locale }

}

intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM); intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE, locale); intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_PREFERENCE, locale); intent.putExtra(RecognizerIntent.EXTRA_ONLY_RETURN_LANGUAGE_PREFERENCE, locale);

public void setToEnglish() { locale = Locale.US; } public void setToSpanish() { locale = new Locale("es", “ES"); }

19

Voice Output Configurationpublic class Voice implements TextToSpeech.OnInitListener {

// More Java

private void setLanguage(Locale language) { // set the reading language locale = language; int result = tts.setLanguage(locale); // Notify the user the language is not on it’s phone if (result == TextToSpeech.LANG_MISSING_DATA || result == TextToSpeech.LANG_NOT_SUPPORTED) { Toast.makeText(context,”This Language is not supported”, Toast.LENGTH_SHORT) .show(); } }

public enum Speed { HALF, NORMAL, ONE_AND_HALF, DOUBLE } public void setSpeechSpeed(Speed speed) { switch (speed) { case HALF: tts.setSpeechRate(0.5f); break; // More case that look very similar } }

public enum Pitch { VERY_LOW, LOW, NORMAL, HIGH, VERY_HIGH } public void setPitch(Pitch pitch) { switch (pitch) { case VERY_LOW: tts.setPitch(0.01f); break;

// More case that look very similar } }

}

20

Voice Output Configurationpublic class

// More Java

private void // set the reading language locale int result = // Notify the user the language is not on it’s phone if (result == TextToSpeech. Toast. . } }

public enum public void switch case // More case that look very similar } }

public enum public void switch case


}

private void setLanguage(Locale language) { // set the reading language locale = language; int result = tts.setLanguage(locale); // Notify the user the language is not on it’s phone if (result == TextToSpeech.LANG_MISSING_DATA || result == TextToSpeech.LANG_NOT_SUPPORTED) { Toast.makeText(context,”This Language is not supported”, Toast.LENGTH_SHORT) .show(); } }

21


// More Java





}

public enum Speed { HALF, NORMAL, ONE_AND_HALF, DOUBLE } public void setSpeechSpeed(Speed speed) { switch (speed) { case HALF: tts.setSpeechRate(0.5f); break; // More case that look very similar } }

22


// More Java





}

public enum Pitch { VERY_LOW, LOW, NORMAL, HIGH, VERY_HIGH } public void setPitch(Pitch pitch) { switch (pitch) { case VERY_LOW: tts.setPitch(0.01f); break;


23

Demo

24

We Are Done!!!

25

Resources• Slides: http://www.slideshare.net/JorgePaez15/introduction-to-voice-io-on-android

• Code: https://github.com/jorgep19/Android_VoiceIO

• Blog post coming next week: https://www.ibm.com/innovation/milab/blog/

http://www.slideshare.net/JorgePaez15/introduction-to-voice-io-on-android

https://github.com/jorgep19/Android_VoiceIO

https://www.ibm.com/innovation/milab/blog/

Thank You

27

Encore!

28

Sending Data from the Watchpublic class MainActivity extends Activity { // More Java

public void onSoundProcessed(Ears.Guess[] guesses) { new TellPhoneTask(gClient).execute(guesses); }

private GoogleApiClient buildGoogleApiClient() { return new GoogleApiClient.Builder(this) .addApi(Wearable.API) // Additional configuration for the Google Client .build(); }

}

public class TellPhoneTask extends AsyncTask<Guess, Void, Boolean> { @Override public Boolean doInBackground(Guess... guesses) { PutDataMapRequest putDataMapReq = PutDataMapRequest.create(Ears.GUESSES_PATH); putDataMapReq.getDataMap().putDataMapArrayList(Ears.GUESSES_KEY, putGuessesInDataMapArray(guesses)); PutDataRequest putRequest = putDataMapReq.asPutDataRequest(); DataApi.DataItemResult result = Wearable.DataApi.putDataItem(gClient, putRequest).await(); return result.getStatus().isSuccess(); } // More Java}

29

Sending Data from the Watchpublic class // More Java

public void new TellPhoneTask(}

private GoogleApiClient buildGoogleApiClient() { return new }

}

public class @Override public PutDataMapRequest putDataMapReq = PutDataMapRequest. putDataMapReq.getDataMap().putDataMapArrayList(Ears. PutDataRequest putRequest = putDataMapReq.asPutDataRequest() DataApi.DataItemResult result = Wearable. return } // More Java}

public void onSoundProcessed(Ears.Guess[] guesses) { new TellPhoneTask(gClient).execute(guesses); }

private GoogleApiClient buildGoogleApiClient() { return new GoogleApiClient.Builder(this) .addApi(Wearable.API) // Additional configuration for the Google Client .build(); }

30

Sending Data from the Watchpublic class // More Java

public void new TellPhoneTask(}

private GoogleApiClient buildGoogleApiClient() { return new }

}

public class @Override public PutDataMapRequest putDataMapReq = PutDataMapRequest. putDataMapReq.getDataMap().putDataMapArrayList(Ears. PutDataRequest putRequest = putDataMapReq.asPutDataRequest() DataApi.DataItemResult result = Wearable. return } // More Java}

public Boolean doInBackground(Guess... guesses) { PutDataMapRequest putDataMapReq = PutDataMapRequest.create(Ears.GUESSES_PATH); putDataMapReq.getDataMap().putDataMapArrayList(Ears.GUESSES_KEY, putGuessesInDataMapArray(guesses)); PutDataRequest putRequest = putDataMapReq.asPutDataRequest(); DataApi.DataItemResult result = Wearable.DataApi.putDataItem(gClient, putRequest).await(); return result.getStatus().isSuccess(); }

31

Receive Data on the Phone<?xml version="1.0" encoding="utf-8"?> <manifest xmlns:android="http://schemas.android.com/apk/res/android" package="com.jpdevs.voiceio" > <application android:allowBackup="true" android:icon="@mipmap/ic_launcher" android:label="@string/app_name" android:supportsRtl="true" android:theme="@style/AppTheme" >  <service android:name=".ListenToWatchService" > <intent-filter> <action android:name="com.google.android.gms.wearable.BIND_LISTENER" /> </intent-filter> </service> </application> </manifest>

public class ListenToWatchService extends WearableListenerService { @Override public void onDataChanged(DataEventBuffer dataEvents) { for (DataEvent event : dataEvents) { if (event.getType() == DataEvent.TYPE_CHANGED) { DataItem item = event.getDataItem(); if (item.getUri().getPath().compareTo(Ears.GUESSES_PATH) == 0) { DataMap dataMap = DataMapItem.fromDataItem(item).getDataMap(); ArrayList<Guess> guesses = parseGuessesDataMap(dataMap); showNotification(guesses); } } } }

// More Java }

32

Receive Data on the Phone<?xml version=<manifest package= <application < < </ </ </application</manifest

public class @Override public void } } }

// More Java}

<service android:name=".ListenToWatchService" > <intent-filter> <action android:name="com.google.android.gms.wearable.BIND_LISTENER" /> </intent-filter> </service>

33

Receive Data on the Phone<?xml version=<manifest package= <application < < </ </ </application</manifest

public class @Override public void } } }

// More Java}

extends WearableListenerService { @Override public void onDataChanged(DataEventBuffer dataEvents) { for (DataEvent event : dataEvents) { if (event.getType() == DataEvent.TYPE_CHANGED) { DataItem item = event.getDataItem(); if (item.getUri().getPath().compareTo(Ears.GUESSES_PATH) == 0) { DataMap dataMap = DataMapItem.fromDataItem(item).getDataMap(); ArrayList<Guess> guesses = parseGuessesDataMap(dataMap); showNotification(guesses); } } } }

Thank You