From b6ed1c047de6f7aa9bae1663d3dc4a84f8ea6da9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 17 Jan 2026 12:06:48 +0000 Subject: [PATCH] Add Android global actions support Co-authored-by: jonasbark <1151304+jonasbark@users.noreply.github.com> --- .../accessibility/AccessibilityApi.kt | 45 ++++++++++++ .../accessibility/AccessibilityPlugin.kt | 5 ++ .../accessibility/AccessibilityService.kt | 15 ++++ .../de/jonasbark/accessibility/Listener.kt | 2 + accessibility/api.dart | 13 ++++ accessibility/lib/accessibility.dart | 40 +++++++++++ lib/pages/button_edit.dart | 71 ++++++++++++++++--- lib/utils/actions/android.dart | 9 +++ lib/utils/keymap/keymap.dart | 46 +++++++++++- 9 files changed, 234 insertions(+), 12 deletions(-) diff --git a/accessibility/android/src/main/kotlin/de/jonasbark/accessibility/AccessibilityApi.kt b/accessibility/android/src/main/kotlin/de/jonasbark/accessibility/AccessibilityApi.kt index 2660844..eca5f5e 100644 --- a/accessibility/android/src/main/kotlin/de/jonasbark/accessibility/AccessibilityApi.kt +++ b/accessibility/android/src/main/kotlin/de/jonasbark/accessibility/AccessibilityApi.kt @@ -90,6 +90,23 @@ enum class MediaAction(val raw: Int) { } } +enum class GlobalAction(val raw: Int) { + BACK(0), + DPAD_CENTER(1), + DOWN(2), + RIGHT(3), + UP(4), + LEFT(5), + HOME(6), + RECENTS(7); + + companion object { + fun ofRaw(raw: Int): GlobalAction? { + return values().firstOrNull { it.raw == raw } + } + } +} + /** Generated class from Pigeon that represents data sent in messages. */ data class WindowEvent ( val packageName: String, @@ -174,6 +191,11 @@ private open class AccessibilityApiPigeonCodec : StandardMessageCodec() { MediaAction.ofRaw(it.toInt()) } } + 132.toByte() -> { + return (readValue(buffer) as Long?)?.let { + GlobalAction.ofRaw(it.toInt()) + } + } 130.toByte() -> { return (readValue(buffer) as? List)?.let { WindowEvent.fromList(it) @@ -193,6 +215,10 @@ private open class AccessibilityApiPigeonCodec : StandardMessageCodec() { stream.write(129) writeValue(stream, value.raw) } + is GlobalAction -> { + stream.write(132) + writeValue(stream, value.raw) + } is WindowEvent -> { stream.write(130) writeValue(stream, value.toList()) @@ -213,6 +239,7 @@ interface Accessibility { fun hasPermission(): Boolean fun openPermissions() fun performTouch(x: Double, y: Double, isKeyDown: Boolean, isKeyUp: Boolean) + fun performGlobalAction(action: GlobalAction) fun controlMedia(action: MediaAction) fun isRunning(): Boolean fun ignoreHidDevices() @@ -279,6 +306,24 @@ interface Accessibility { channel.setMessageHandler(null) } } + run { + val channel = BasicMessageChannel(binaryMessenger, "dev.flutter.pigeon.accessibility.Accessibility.performGlobalAction$separatedMessageChannelSuffix", codec) + if (api != null) { + channel.setMessageHandler { message, reply -> + val args = message as List + val actionArg = args[0] as GlobalAction + val wrapped: List = try { + api.performGlobalAction(actionArg) + listOf(null) + } catch (exception: Throwable) { + AccessibilityApiPigeonUtils.wrapError(exception) + } + reply.reply(wrapped) + } + } else { + channel.setMessageHandler(null) + } + } run { val channel = BasicMessageChannel(binaryMessenger, "dev.flutter.pigeon.accessibility.Accessibility.controlMedia$separatedMessageChannelSuffix", codec) if (api != null) { diff --git a/accessibility/android/src/main/kotlin/de/jonasbark/accessibility/AccessibilityPlugin.kt b/accessibility/android/src/main/kotlin/de/jonasbark/accessibility/AccessibilityPlugin.kt index 0f369bb..0117a22 100644 --- a/accessibility/android/src/main/kotlin/de/jonasbark/accessibility/AccessibilityPlugin.kt +++ b/accessibility/android/src/main/kotlin/de/jonasbark/accessibility/AccessibilityPlugin.kt @@ -2,6 +2,7 @@ package de.jonasbark.accessibility import AKeyEvent import Accessibility +import GlobalAction import HidKeyPressedStreamHandler import MediaAction import PigeonEventSink @@ -66,6 +67,10 @@ class AccessibilityPlugin: FlutterPlugin, Accessibility { Observable.toService?.performTouch(x = x, y = y, isKeyUp = isKeyUp, isKeyDown = isKeyDown) ?: error("Service not running") } + override fun performGlobalAction(action: GlobalAction) { + Observable.toService?.performGlobalAction(action) ?: error("Service not running") + } + override fun controlMedia(action: MediaAction) { val audioService = context.getSystemService(Context.AUDIO_SERVICE) as android.media.AudioManager when (action) { diff --git a/accessibility/android/src/main/kotlin/de/jonasbark/accessibility/AccessibilityService.kt b/accessibility/android/src/main/kotlin/de/jonasbark/accessibility/AccessibilityService.kt index 4aaa407..cec4a9e 100644 --- a/accessibility/android/src/main/kotlin/de/jonasbark/accessibility/AccessibilityService.kt +++ b/accessibility/android/src/main/kotlin/de/jonasbark/accessibility/AccessibilityService.kt @@ -15,6 +15,7 @@ import android.view.KeyEvent import android.view.ViewConfiguration import android.view.accessibility.AccessibilityEvent import android.view.accessibility.AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED +import GlobalAction class AccessibilityService : AccessibilityService(), Listener { @@ -97,6 +98,20 @@ class AccessibilityService : AccessibilityService(), Listener { } } + override fun performGlobalAction(action: GlobalAction) { + val mappedAction = when (action) { + GlobalAction.BACK -> android.accessibilityservice.AccessibilityService.GLOBAL_ACTION_BACK + GlobalAction.DPAD_CENTER -> android.accessibilityservice.AccessibilityService.GLOBAL_ACTION_DPAD_CENTER + GlobalAction.DOWN -> android.accessibilityservice.AccessibilityService.GLOBAL_ACTION_DPAD_DOWN + GlobalAction.RIGHT -> android.accessibilityservice.AccessibilityService.GLOBAL_ACTION_DPAD_RIGHT + GlobalAction.UP -> android.accessibilityservice.AccessibilityService.GLOBAL_ACTION_DPAD_UP + GlobalAction.LEFT -> android.accessibilityservice.AccessibilityService.GLOBAL_ACTION_DPAD_LEFT + GlobalAction.HOME -> android.accessibilityservice.AccessibilityService.GLOBAL_ACTION_HOME + GlobalAction.RECENTS -> android.accessibilityservice.AccessibilityService.GLOBAL_ACTION_RECENTS + } + performGlobalAction(mappedAction) + } + override fun performTouch(x: Double, y: Double, isKeyDown: Boolean, isKeyUp: Boolean) { val gestureBuilder = GestureDescription.Builder() val path = Path() diff --git a/accessibility/android/src/main/kotlin/de/jonasbark/accessibility/Listener.kt b/accessibility/android/src/main/kotlin/de/jonasbark/accessibility/Listener.kt index dec1262..a92a8e9 100644 --- a/accessibility/android/src/main/kotlin/de/jonasbark/accessibility/Listener.kt +++ b/accessibility/android/src/main/kotlin/de/jonasbark/accessibility/Listener.kt @@ -2,6 +2,7 @@ package de.jonasbark.accessibility import android.graphics.Rect import android.view.KeyEvent +import GlobalAction import java.util.concurrent.ConcurrentHashMap object Observable { @@ -15,6 +16,7 @@ object Observable { interface Listener { fun performTouch(x: Double, y: Double, isKeyDown: Boolean, isKeyUp: Boolean) + fun performGlobalAction(action: GlobalAction) } interface Receiver { diff --git a/accessibility/api.dart b/accessibility/api.dart index 011885b..b10cf03 100644 --- a/accessibility/api.dart +++ b/accessibility/api.dart @@ -8,6 +8,8 @@ abstract class Accessibility { void performTouch(double x, double y, {bool isKeyDown = true, bool isKeyUp = false}); + void performGlobalAction(GlobalAction action); + void controlMedia(MediaAction action); bool isRunning(); @@ -19,6 +21,17 @@ abstract class Accessibility { enum MediaAction { playPause, next, volumeUp, volumeDown } +enum GlobalAction { + back, + dpadCenter, + down, + right, + up, + left, + home, + recents, +} + class WindowEvent { final String packageName; final int top; diff --git a/accessibility/lib/accessibility.dart b/accessibility/lib/accessibility.dart index 9292074..d1670c1 100644 --- a/accessibility/lib/accessibility.dart +++ b/accessibility/lib/accessibility.dart @@ -36,6 +36,17 @@ enum MediaAction { volumeDown, } +enum GlobalAction { + back, + dpadCenter, + down, + right, + up, + left, + home, + recents, +} + class WindowEvent { WindowEvent({ required this.packageName, @@ -164,6 +175,9 @@ class _PigeonCodec extends StandardMessageCodec { } else if (value is MediaAction) { buffer.putUint8(129); writeValue(buffer, value.index); + } else if (value is GlobalAction) { + buffer.putUint8(132); + writeValue(buffer, value.index); } else if (value is WindowEvent) { buffer.putUint8(130); writeValue(buffer, value.encode()); @@ -181,6 +195,9 @@ class _PigeonCodec extends StandardMessageCodec { case 129: final int? value = readValue(buffer) as int?; return value == null ? null : MediaAction.values[value]; + case 132: + final int? value = readValue(buffer) as int?; + return value == null ? null : GlobalAction.values[value]; case 130: return WindowEvent.decode(readValue(buffer)!); case 131: @@ -280,6 +297,29 @@ class Accessibility { } } + Future performGlobalAction(GlobalAction action) async { + final String pigeonVar_channelName = 'dev.flutter.pigeon.accessibility.Accessibility.performGlobalAction$pigeonVar_messageChannelSuffix'; + final BasicMessageChannel pigeonVar_channel = BasicMessageChannel( + pigeonVar_channelName, + pigeonChannelCodec, + binaryMessenger: pigeonVar_binaryMessenger, + ); + final Future pigeonVar_sendFuture = pigeonVar_channel.send([action]); + final List? pigeonVar_replyList = + await pigeonVar_sendFuture as List?; + if (pigeonVar_replyList == null) { + throw _createConnectionError(pigeonVar_channelName); + } else if (pigeonVar_replyList.length > 1) { + throw PlatformException( + code: pigeonVar_replyList[0]! as String, + message: pigeonVar_replyList[1] as String?, + details: pigeonVar_replyList[2], + ); + } else { + return; + } + } + Future controlMedia(MediaAction action) async { final String pigeonVar_channelName = 'dev.flutter.pigeon.accessibility.Accessibility.controlMedia$pigeonVar_messageChannelSuffix'; final BasicMessageChannel pigeonVar_channel = BasicMessageChannel( diff --git a/lib/pages/button_edit.dart b/lib/pages/button_edit.dart index 730ec8c..d709147 100644 --- a/lib/pages/button_edit.dart +++ b/lib/pages/button_edit.dart @@ -201,6 +201,7 @@ class _ButtonEditPageState extends State { _keyPair.isLongPress = keyPairAction.isLongPress; _keyPair.inGameAction = keyPairAction.inGameAction; _keyPair.inGameActionValue = keyPairAction.inGameActionValue; + _keyPair.androidAction = null; setState(() {}); }, child: Text(keyPairAction.toString()), @@ -227,6 +228,7 @@ class _ButtonEditPageState extends State { keyPair: _keyPair, ), ); + _keyPair.androidAction = null; setState(() {}); widget.onUpdate(); }, @@ -243,6 +245,7 @@ class _ButtonEditPageState extends State { } _keyPair.physicalKey = null; _keyPair.logicalKey = null; + _keyPair.androidAction = null; await Navigator.of(context).push( MaterialPageRoute( builder: (c) => TouchAreaSetupPage( @@ -272,6 +275,7 @@ class _ButtonEditPageState extends State { onPressed: (c) { _keyPair.physicalKey = PhysicalKeyboardKey.mediaPlayPause; _keyPair.logicalKey = null; + _keyPair.androidAction = null; setState(() {}); widget.onUpdate(); @@ -283,6 +287,7 @@ class _ButtonEditPageState extends State { onPressed: (c) { _keyPair.physicalKey = PhysicalKeyboardKey.mediaStop; _keyPair.logicalKey = null; + _keyPair.androidAction = null; setState(() {}); widget.onUpdate(); @@ -294,6 +299,7 @@ class _ButtonEditPageState extends State { onPressed: (c) { _keyPair.physicalKey = PhysicalKeyboardKey.mediaTrackPrevious; _keyPair.logicalKey = null; + _keyPair.androidAction = null; setState(() {}); widget.onUpdate(); @@ -305,6 +311,7 @@ class _ButtonEditPageState extends State { onPressed: (c) { _keyPair.physicalKey = PhysicalKeyboardKey.mediaTrackNext; _keyPair.logicalKey = null; + _keyPair.androidAction = null; setState(() {}); widget.onUpdate(); @@ -316,6 +323,7 @@ class _ButtonEditPageState extends State { onPressed: (c) { _keyPair.physicalKey = PhysicalKeyboardKey.audioVolumeUp; _keyPair.logicalKey = null; + _keyPair.androidAction = null; setState(() {}); widget.onUpdate(); @@ -328,6 +336,7 @@ class _ButtonEditPageState extends State { onPressed: (c) { _keyPair.physicalKey = PhysicalKeyboardKey.audioVolumeDown; _keyPair.logicalKey = null; + _keyPair.androidAction = null; setState(() {}); widget.onUpdate(); @@ -339,6 +348,43 @@ class _ButtonEditPageState extends State { }, ), ), + if (core.logic.showLocalControl && + core.settings.getLocalEnabled() && + core.actionHandler is AndroidActions) + Builder( + builder: (context) => SelectableCard( + icon: Icons.settings_remote_outlined, + isActive: _keyPair.androidAction != null, + title: Text('Android System Action'), + value: _keyPair.androidAction?.title, + onPressed: () { + showDropdown( + context: context, + builder: (c) => DropdownMenu( + children: AndroidSystemAction.values + .map( + (action) => MenuButton( + leading: Icon(action.icon), + onPressed: (_) { + _keyPair.androidAction = action; + _keyPair.physicalKey = null; + _keyPair.logicalKey = null; + _keyPair.modifiers = []; + _keyPair.touchPosition = Offset.zero; + _keyPair.inGameAction = null; + _keyPair.inGameActionValue = null; + setState(() {}); + widget.onUpdate(); + }, + child: Text(action.title), + ), + ) + .toList(), + ), + ); + }, + ), + ), ], if (core.connection.accessories.isNotEmpty) ...[ @@ -368,6 +414,7 @@ class _ButtonEditPageState extends State { onPressed: (_) { _keyPair.inGameAction = InGameAction.headwindSpeed; _keyPair.inGameActionValue = value; + _keyPair.androidAction = null; widget.onUpdate(); setState(() {}); }, @@ -381,6 +428,7 @@ class _ButtonEditPageState extends State { onPressed: (_) { _keyPair.inGameAction = InGameAction.headwindHeartRateMode; _keyPair.inGameActionValue = null; + _keyPair.androidAction = null; widget.onUpdate(); setState(() {}); }, @@ -415,6 +463,7 @@ class _ButtonEditPageState extends State { _keyPair.touchPosition = Offset.zero; _keyPair.inGameAction = null; _keyPair.inGameActionValue = null; + _keyPair.androidAction = null; widget.onUpdate(); setState(() {}); }, @@ -450,16 +499,17 @@ class _ButtonEditPageState extends State { children: action.possibleValues!.map( (ingame) { return MenuButton( - child: Text(ingame.toString()), - onPressed: (_) { - _keyPair.touchPosition = Offset.zero; - _keyPair.physicalKey = null; - _keyPair.logicalKey = null; - _keyPair.inGameAction = action; - _keyPair.inGameActionValue = ingame; - widget.onUpdate(); - setState(() {}); - }, + child: Text(ingame.toString()), + onPressed: (_) { + _keyPair.touchPosition = Offset.zero; + _keyPair.physicalKey = null; + _keyPair.logicalKey = null; + _keyPair.androidAction = null; + _keyPair.inGameAction = action; + _keyPair.inGameActionValue = ingame; + widget.onUpdate(); + setState(() {}); + }, ); }, ).toList(), @@ -469,6 +519,7 @@ class _ButtonEditPageState extends State { _keyPair.touchPosition = Offset.zero; _keyPair.physicalKey = null; _keyPair.logicalKey = null; + _keyPair.androidAction = null; _keyPair.inGameAction = action; _keyPair.inGameActionValue = null; widget.onUpdate(); diff --git a/lib/utils/actions/android.dart b/lib/utils/actions/android.dart index 36df964..d7405ad 100644 --- a/lib/utils/actions/android.dart +++ b/lib/utils/actions/android.dart @@ -79,6 +79,15 @@ class AndroidActions extends BaseActions { return Success("Key pressed: ${keyPair.toString()}"); } + if (keyPair.androidAction != null) { + if (!core.settings.getLocalEnabled() || !core.logic.showLocalControl || !isKeyDown) { + return Ignored('Global action ignored'); + } + await accessibilityHandler.performGlobalAction(keyPair.androidAction!.globalAction); + await IAPManager.instance.incrementCommandCount(); + return Success("Global action: ${keyPair.androidAction!.title}"); + } + final point = await resolveTouchPosition(keyPair: keyPair, windowInfo: windowInfo); if (point != Offset.zero) { try { diff --git a/lib/utils/keymap/keymap.dart b/lib/utils/keymap/keymap.dart index c895055..63b0b8a 100644 --- a/lib/utils/keymap/keymap.dart +++ b/lib/utils/keymap/keymap.dart @@ -1,6 +1,7 @@ import 'dart:async'; import 'dart:convert'; +import 'package:accessibility/accessibility.dart'; import 'package:bike_control/gen/l10n.dart'; import 'package:bike_control/main.dart'; import 'package:bike_control/utils/actions/android.dart'; @@ -13,6 +14,23 @@ import 'package:shadcn_flutter/shadcn_flutter.dart'; import '../actions/base_actions.dart'; import 'apps/custom_app.dart'; +enum AndroidSystemAction { + back('Back', Icons.arrow_back, GlobalAction.back), + dpadCenter('DPAD Center', Icons.radio_button_checked_outlined, GlobalAction.dpadCenter), + down('DPAD Down', Icons.arrow_downward, GlobalAction.down), + right('DPAD Right', Icons.arrow_forward, GlobalAction.right), + up('DPAD Up', Icons.arrow_upward, GlobalAction.up), + left('DPAD Left', Icons.keyboard_arrow_left, GlobalAction.left), + home('Home', Icons.home_outlined, GlobalAction.home), + recents('Recents', Icons.apps, GlobalAction.recents); + + final String title; + final IconData icon; + final GlobalAction globalAction; + + const AndroidSystemAction(this.title, this.icon, this.globalAction); +} + class Keymap { static Keymap custom = Keymap(keyPairs: []); @@ -50,6 +68,7 @@ class Keymap { keyPair.isLongPress = false; keyPair.inGameAction = null; keyPair.inGameActionValue = null; + keyPair.androidAction = null; } _updateStream.add(null); } @@ -112,6 +131,7 @@ class KeyPair { bool isLongPress; InGameAction? inGameAction; int? inGameActionValue; + AndroidSystemAction? androidAction; KeyPair({ required this.buttons, @@ -122,6 +142,7 @@ class KeyPair { this.isLongPress = false, this.inGameAction, this.inGameActionValue, + this.androidAction, }); bool get isSpecialKey => @@ -146,6 +167,11 @@ class KeyPair { //_ when inGameAction != null && core.logic.emulatorEnabled => Icons.link, _ when inGameAction != null && inGameAction!.icon != null => inGameAction!.icon, + _ when androidAction != null && + core.logic.showLocalControl && + core.settings.getLocalEnabled() && + core.actionHandler is AndroidActions => + androidAction!.icon, _ when physicalKey != null && core.actionHandler.supportedModes.contains(SupportedMode.keyboard) => RadixIcons.keyboard, _ @@ -159,7 +185,11 @@ class KeyPair { } bool get hasNoAction => - logicalKey == null && physicalKey == null && touchPosition == Offset.zero && inGameAction == null; + logicalKey == null && + physicalKey == null && + touchPosition == Offset.zero && + inGameAction == null && + androidAction == null; bool get hasActiveAction => screenshotMode || @@ -167,6 +197,10 @@ class KeyPair { core.logic.showLocalControl && core.settings.getLocalEnabled() && core.actionHandler.supportedModes.contains(SupportedMode.keyboard)) || + (androidAction != null && + core.logic.showLocalControl && + core.settings.getLocalEnabled() && + core.actionHandler is AndroidActions) || (touchPosition != Offset.zero && core.logic.showLocalRemoteOptions && core.actionHandler.supportedModes.contains(SupportedMode.touch)) || @@ -193,6 +227,8 @@ class KeyPair { inGameAction!.title, if (inGameActionValue != null) '$inGameActionValue', ].joinToString(separator: ': ') + : (androidAction != null && core.logic.showLocalControl && core.actionHandler is AndroidActions) + ? androidAction!.title : (isSpecialKey && core.actionHandler.supportedModes.contains(SupportedMode.media)) ? switch (physicalKey) { PhysicalKeyboardKey.mediaPlayPause => AppLocalizations.current.playPause, @@ -247,6 +283,7 @@ class KeyPair { 'isLongPress': isLongPress, 'inGameAction': inGameAction?.name, 'inGameActionValue': inGameActionValue, + 'androidAction': androidAction?.name, }); } @@ -295,6 +332,9 @@ class KeyPair { ? InGameAction.values.firstOrNullWhere((element) => element.name == decoded['inGameAction']) : null, inGameActionValue: decoded['inGameActionValue'], + androidAction: decoded.containsKey('androidAction') + ? AndroidSystemAction.values.firstOrNullWhere((element) => element.name == decoded['androidAction']) + : null, ); } @@ -309,7 +349,8 @@ class KeyPair { touchPosition == other.touchPosition && isLongPress == other.isLongPress && inGameAction == other.inGameAction && - inGameActionValue == other.inGameActionValue; + inGameActionValue == other.inGameActionValue && + androidAction == other.androidAction; @override int get hashCode => Object.hash( @@ -320,5 +361,6 @@ class KeyPair { isLongPress, inGameAction, inGameActionValue, + androidAction, ); }