Skip to content

Commit

Permalink
Added word substitution settings for voice control.
Browse files Browse the repository at this point in the history
  • Loading branch information
AndreyPavlenko committed Oct 23, 2021
1 parent 4a2c575 commit 56bcbd4
Show file tree
Hide file tree
Showing 13 changed files with 214 additions and 47 deletions.
4 changes: 2 additions & 2 deletions build.gradle
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
ext {
def abi = project.properties['ABI']
VERSION_CODE = 167
VERSION_NAME = "1.8.4"
VERSION_CODE = 169
VERSION_NAME = "1.8.5"
SDK_MIN_VERSION = 23
SDK_TARGET_VERSION = 30
SDK_COMPILE_VERSION = 30
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import static me.aap.fermata.media.pref.PlaybackControlPrefs.PREV_VOICE_CONTROl;
import static me.aap.fermata.ui.activity.MainActivityPrefs.BRIGHTNESS;
import static me.aap.fermata.ui.activity.MainActivityPrefs.CHANGE_BRIGHTNESS;
import static me.aap.fermata.ui.activity.MainActivityPrefs.VOICE_CONTROL_SUBST;
import static me.aap.fermata.ui.activity.MainActivityPrefs.VOICE_CONTROl_ENABLED;
import static me.aap.fermata.ui.activity.MainActivityPrefs.VOICE_CONTROl_FB;
import static me.aap.fermata.ui.activity.MainActivityPrefs.VOICE_CONTROl_M;
Expand Down Expand Up @@ -47,6 +48,7 @@
import android.view.KeyEvent;
import android.view.MotionEvent;
import android.view.View;
import android.widget.EditText;
import android.widget.Toast;

import androidx.annotation.LayoutRes;
Expand Down Expand Up @@ -140,7 +142,7 @@ public class MainActivityDelegate extends ActivityDelegate implements
private boolean exitPressed;
private int brightness = 255;
private SpeechListener speechListener;
private VoiceSearchHandler voiceSearchHandler;
private VoiceCommandHandler voiceCommandHandler;

public MainActivityDelegate(AppActivity activity, FermataServiceUiBinder binder) {
super(activity);
Expand Down Expand Up @@ -608,27 +610,35 @@ public OverlayMenu getToolBarMenu() {
}

public void startVoiceSearch() {
View focus = getCurrentFocus();
FutureSupplier<int[]> check = isCarActivity()
? completed(new int[]{PERMISSION_GRANTED})
: getAppActivity().checkPermissions(Manifest.permission.RECORD_AUDIO);
check.onCompletion((r, err) -> {
if ((err == null) && (r[0] == PERMISSION_GRANTED)) {
voiceSearch();
voiceSearch(focus);
return;
}
if (err != null) Log.e(err, "Failed to request RECORD_AUDIO permission");
UiUtils.showAlert(getContext(), R.string.err_no_audio_record_perm);
});
}

private void voiceSearch() {
VoiceSearchHandler h = voiceSearchHandler;
if (h == null) h = voiceSearchHandler = new VoiceSearchHandler(this);
private void voiceSearch(View focus) {
Intent i = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH);
i.putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true);
i.putExtra(RecognizerIntent.EXTRA_LANGUAGE, Locale.getDefault());
i.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM);
startSpeechRecognizer(i).onSuccess(h::handle);
startSpeechRecognizer(i).onSuccess(q -> {
if (focus instanceof EditText) {
((EditText) focus).setText(q.get(0));
focus.requestFocus();
} else {
VoiceCommandHandler h = voiceCommandHandler;
if (h == null) h = voiceCommandHandler = new VoiceCommandHandler(this);
h.handle(q);
}
});
}

private FutureSupplier<List<String>> startSpeechRecognizer(Intent i) {
Expand Down Expand Up @@ -659,6 +669,18 @@ public FutureSupplier<PlayableItem> getNextPlayable(Item i) {
: MediaSessionCallbackAssistant.super.getNextPlayable(i);
}

@Override
public EditText createEditText(Context ctx) {
EditText t = getAppActivity().createEditText(ctx);
if (isCarActivity() && getPrefs().getVoiceControlEnabledPref()) {
t.setOnLongClickListener(v -> {
startVoiceSearch();
return true;
});
}
return t;
}

@Override
public DialogBuilder createDialogBuilder(Context ctx) {
return DialogBuilder.create(getContextMenu());
Expand Down Expand Up @@ -838,6 +860,8 @@ public void onPreferenceChanged(PreferenceStore store, List<PreferenceStore.Pref
e.setBooleanPref(PREV_VOICE_CONTROl, false);
}
});
} else if (prefs.contains(VOICE_CONTROL_SUBST)) {
if (voiceCommandHandler != null) voiceCommandHandler.updateWordSubst();
}
}

Expand All @@ -849,6 +873,7 @@ public boolean onKeyDown(int code, KeyEvent event, IntObjectFunction<KeyEvent, B
onBackPressed();
return true;
case KeyEvent.KEYCODE_M:
if (getCurrentFocus() instanceof EditText) return super.onKeyDown(code, event, next);
case KeyEvent.KEYCODE_MENU:
if (getPrefs().getVoiceControlMenuPref()) event.startTracking();
return true;
Expand Down Expand Up @@ -879,6 +904,7 @@ public boolean onKeyUp(int code, KeyEvent event, IntObjectFunction<KeyEvent, Boo
if ((event.getFlags() & KeyEvent.FLAG_CANCELED_LONG_PRESS) == 0) {
switch (code) {
case KeyEvent.KEYCODE_M:
if (getCurrentFocus() instanceof EditText) return super.onKeyUp(code, event, next);
case KeyEvent.KEYCODE_MENU:
if (event.isShiftPressed()) {
getNavBarMediator().showMenu(this);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ public interface MainActivityPrefs extends SharedPreferenceStore, EventBroadcast
Pref<BooleanSupplier> VOICE_CONTROl_ENABLED = Pref.b("VOICE_CONTROl_ENABLED", false);
Pref<BooleanSupplier> VOICE_CONTROl_FB = Pref.b("VOICE_CONTROl_FB", false);
Pref<BooleanSupplier> VOICE_CONTROl_M = Pref.b("VOICE_CONTROl_M", false);
Pref<Supplier<String>> VOICE_CONTROL_SUBST = Pref.s("VOICE_CONTROL_SUBST", "");

Pref<IntSupplier> THEME_AA = Pref.i("THEME_AA", THEME_DARK);
Pref<BooleanSupplier> HIDE_BARS_AA = AUTO ? Pref.b("HIDE_BARS_AA", false) : null;
Expand Down
38 changes: 38 additions & 0 deletions fermata/src/main/java/me/aap/fermata/ui/activity/VoiceCommand.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package me.aap.fermata.ui.activity;

/**
* @author Andrey Pavlenko
*/
public class VoiceCommand {
public static final int ACTION_PLAY = 1;
public static final int ACTION_FIND = 2;
public static final int ACTION_OPEN = 4;

private final String query;
private final int action;

public VoiceCommand(String query, int action) {
this.query = query;
this.action = action;
}

public String getQuery() {
return query;
}

public int getAction() {
return action;
}

public boolean isPlay() {
return (getAction() & ACTION_PLAY) != 0;
}

public boolean isFind() {
return (getAction() & ACTION_FIND) != 0;
}

public boolean isOpen() {
return (getAction() & ACTION_OPEN) != 0;
}
}
Original file line number Diff line number Diff line change
@@ -1,14 +1,21 @@
package me.aap.fermata.ui.activity;

import static me.aap.fermata.media.pref.PlaybackControlPrefs.TIME_UNIT_SECOND;
import static me.aap.fermata.ui.activity.MainActivityPrefs.VOICE_CONTROL_SUBST;
import static me.aap.utils.ui.UiUtils.ID_NULL;

import android.content.res.Resources;

import androidx.annotation.IdRes;
import androidx.annotation.StringRes;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.StringReader;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

Expand All @@ -19,11 +26,12 @@
import me.aap.fermata.ui.fragment.MediaLibFragment;
import me.aap.utils.log.Log;
import me.aap.utils.text.PatternCompat;
import me.aap.utils.text.SharedTextBuilder;

/**
* @author Andrey Pavlenko
*/
class VoiceSearchHandler {
class VoiceCommandHandler {
private static final String ACTION = "A";
private static final String LOCATION = "L";
private static final String QUERY = "Q";
Expand All @@ -42,16 +50,16 @@ class VoiceSearchHandler {
private final Pattern lTV;
private final Pattern lYoutube;
private final Pattern lBrowser;
private final Pattern uSecond;
private final Pattern uMinute;
private final Pattern uHour;
private final Pattern cCurTrack;
private final PatternCompat cFF;
private final PatternCompat cRW;
private final PatternCompat cFindPlayOpen;
private final String[] nums;
private Map<String, String> subst = Collections.emptyMap();

VoiceSearchHandler(MainActivityDelegate activity) {
VoiceCommandHandler(MainActivityDelegate activity) {
this.activity = activity;
Resources res = activity.getContext().getResources();
aFind = compile(res, R.string.vcmd_action_find);
Expand All @@ -66,22 +74,22 @@ class VoiceSearchHandler {
lTV = compile(res, R.string.vcmd_location_tv);
lYoutube = compile(res, R.string.vcmd_location_youtube);
lBrowser = compile(res, R.string.vcmd_location_browser);
uSecond = compile(res, R.string.vcmd_time_unit_second);
uMinute = compile(res, R.string.vcmd_time_unit_minute);
uHour = compile(res, R.string.vcmd_time_unit_hour);
cCurTrack = compile(res, R.string.vcmd_cur_track);
cFF = PatternCompat.compile(res.getString(R.string.vcmd_ff));
cRW = PatternCompat.compile(res.getString(R.string.vcmd_rw));
cFindPlayOpen = PatternCompat.compile(res.getString(R.string.vcmd_find_play_open));
nums = res.getString(R.string.vcmd_nums).split(" ");
updateWordSubst();
}

public boolean handle(List<String> cmd) {
return !cmd.isEmpty() && handle(cmd.get(0));
}

public boolean handle(String cmd) {
cmd = cmd.trim().toLowerCase();
cmd = subst(cmd);

if (aPlay.matcher(cmd).matches()) {
activity.getMediaSessionCallback().play().thenRun(activity::goToCurrent);
Expand Down Expand Up @@ -125,14 +133,16 @@ public boolean handle(String cmd) {
if ((m = cFindPlayOpen.matcher(cmd)).matches()) {
String q = cFindPlayOpen.group(m, QUERY);
if ((q == null) || (q.trim().isEmpty())) return false;
boolean play = !matches(aFind, cFindPlayOpen.group(m, ACTION))
&& !matches(aOpen, cFindPlayOpen.group(m, ACTION));
int action = matches(aFind, cFindPlayOpen.group(m, ACTION)) ? VoiceCommand.ACTION_FIND :
matches(aOpen, cFindPlayOpen.group(m, ACTION)) ? VoiceCommand.ACTION_OPEN :
VoiceCommand.ACTION_PLAY;
VoiceCommand vcmd = new VoiceCommand(q, action);
String location = cFindPlayOpen.group(m, LOCATION);

if (location == null) {
MainActivityFragment f = activity.getActiveMainActivityFragment();
if ((f == null) || !f.isVoiceSearchSupported()) return false;
f.voiceSearch(q, play);
if ((f == null) || !f.isVoiceCommandsSupported()) return false;
f.voiceCommand(vcmd);
return true;
}

Expand All @@ -152,34 +162,34 @@ else if (amgr.hasAddon(R.id.web_browser_fragment) && matches(lBrowser, location)

if (fid == ID_NULL) return false;
activity.showFragment(fid);
searchInFragment(fid, q, play, 0);
searchInFragment(fid, vcmd, 0);
return true;
}

return false;
}

private void searchInFragment(@IdRes int id, String q, boolean play, int attempt) {
private void searchInFragment(@IdRes int id, VoiceCommand cmd, int attempt) {
if (attempt == 100) {
Log.e("Failed to perform search in fragment ", id);
return;
}
MainActivityFragment f = activity.getActiveMainActivityFragment();
if ((f == null) || (f.getFragmentId() != id)) {
activity.getHandler().post(() -> searchInFragment(id, q, play, attempt + 1));
activity.getHandler().post(() -> searchInFragment(id, cmd, attempt + 1));
} else if ((f.getFragmentId() == R.id.folders_fragment)
|| (f.getFragmentId() == R.id.playlists_fragment)) {
MediaLibFragment mf = (MediaLibFragment) f;
mf.findFolder(q).onSuccess(folder -> {
mf.findFolder(cmd.getQuery()).onSuccess(folder -> {
if (folder == null) {
f.voiceSearch(q, play);
f.voiceCommand(cmd);
} else {
if (play) mf.playFolder(folder);
if (cmd.isPlay()) mf.playFolder(folder);
else mf.openFolder(folder);
}
});
} else {
f.voiceSearch(q, play);
f.voiceCommand(cmd);
}
}

Expand All @@ -189,10 +199,57 @@ private void playFavorites(int attempt) {
return;
}
MainActivityFragment f = activity.getActiveMainActivityFragment();
if (f.getFragmentId() == R.id.favorites_fragment) ((FavoritesFragment) f).play();
if (f instanceof FavoritesFragment) ((FavoritesFragment) f).play();
else activity.getHandler().post(() -> playFavorites(attempt + 1));
}

public void updateWordSubst() {
String pref = activity.getPrefs().getStringPref(VOICE_CONTROL_SUBST);
Map<String, String> subst = new HashMap<>();
try (BufferedReader r = new BufferedReader(new StringReader(pref))) {
for (String l = r.readLine(); l != null; l = r.readLine()) {
int idx = l.indexOf(':');
if (idx < 0) continue;
subst.put(l.substring(0, idx).trim().toLowerCase(),
l.substring(idx + 1).trim().toLowerCase());
}
} catch (IOException ignore) {
}
this.subst = subst.isEmpty() ? Collections.emptyMap() : subst;
}

private String subst(String cmd) {
Map<String, String> subst = this.subst;
int start = -1;

try (SharedTextBuilder b = SharedTextBuilder.get()) {
for (int i = 0, n = cmd.codePointCount(0, cmd.length()); i < n; i++) {
int c = cmd.codePointAt(i);

if (Character.isWhitespace(c)) {
if (start == -1) continue;
if (!subst.isEmpty()) {
String r = subst.get(b.substring(start, b.length()));
if (r != null) b.replace(start, b.length(), r);
}
start = -1;
} else {
if (start == -1) {
if (b.length() != 0) b.append(' ');
start = b.length();
}
b.appendCodePoint(Character.toLowerCase(c));
}
}
if ((start != -1) && !subst.isEmpty()) {
String r = subst.get(b.substring(start, b.length()));
if (r != null) b.replace(start, b.length(), r);
}

return b.toString();
}
}

private static boolean matches(Pattern p, String s) {
return (s != null) && p.matcher(s).matches();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import androidx.annotation.NonNull;

import me.aap.fermata.ui.activity.MainActivityDelegate;
import me.aap.fermata.ui.activity.VoiceCommand;
import me.aap.utils.ui.fragment.ActivityFragment;
import me.aap.utils.ui.menu.OverlayMenu;
import me.aap.utils.ui.view.FloatingButton;
Expand Down Expand Up @@ -47,10 +48,10 @@ public void contributeToNavBarMenu(OverlayMenu.Builder builder) {
public void discardSelection() {
}

public boolean isVoiceSearchSupported() {
public boolean isVoiceCommandsSupported() {
return false;
}

public void voiceSearch(@NonNull String query, boolean play) {
public void voiceCommand(VoiceCommand cmd) {
}
}
Loading

0 comments on commit 56bcbd4

Please sign in to comment.