diff --git a/ChatApp/app/src/androidTest/kotlin/com/example/chatapp/appfunctions/AppFunctionInstrumentationTest.kt b/ChatApp/app/src/androidTest/kotlin/com/example/chatapp/appfunctions/AppFunctionInstrumentationTest.kt index 74e7e8e..4678127 100644 --- a/ChatApp/app/src/androidTest/kotlin/com/example/chatapp/appfunctions/AppFunctionInstrumentationTest.kt +++ b/ChatApp/app/src/androidTest/kotlin/com/example/chatapp/appfunctions/AppFunctionInstrumentationTest.kt @@ -24,7 +24,6 @@ import androidx.appfunctions.ExecuteAppFunctionRequest import androidx.appfunctions.ExecuteAppFunctionResponse import androidx.test.core.app.ApplicationProvider import androidx.test.ext.junit.runners.AndroidJUnit4 -import com.example.chatapp.appfunctions.ChatAppFunctionService import com.example.chatapp.data.MessageRepository import com.example.chatapp.data.RecipientsRepository import com.google.common.truth.Truth.assertThat diff --git a/ChatApp/app/src/main/kotlin/com/example/chatapp/InputBar.kt b/ChatApp/app/src/main/kotlin/com/example/chatapp/InputBar.kt index 2c8c382..3cc9724 100644 --- a/ChatApp/app/src/main/kotlin/com/example/chatapp/InputBar.kt +++ b/ChatApp/app/src/main/kotlin/com/example/chatapp/InputBar.kt @@ -38,6 +38,7 @@ import androidx.compose.material.icons.filled.Add import androidx.compose.material3.FilledIconButton import androidx.compose.material3.Icon import androidx.compose.material3.IconButton +import androidx.compose.material3.IconButtonDefaults import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Surface import androidx.compose.material3.Text @@ -77,7 +78,8 @@ internal fun InputBar( Surface( modifier = modifier, - tonalElevation = 3.dp, + color = Color.Transparent, + tonalElevation = 0.dp, ) { Column { if (selectedImages.isNotEmpty()) { @@ -107,18 +109,25 @@ internal fun InputBar( verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(4.dp), ) { - IconButton( + FilledIconButton( onClick = { photoPickerLauncher.launch( PickVisualMediaRequest(ActivityResultContracts.PickVisualMedia.ImageOnly), ) }, + modifier = Modifier.size(56.dp), + colors = + IconButtonDefaults.filledIconButtonColors( + containerColor = MaterialTheme.colorScheme.surfaceVariant, + contentColor = MaterialTheme.colorScheme.onSurfaceVariant, + ), ) { Icon( imageVector = Icons.Default.Add, contentDescription = "Add Image", ) } + Spacer(modifier = Modifier.width(4.dp)) TextField( value = value, onValueChange = onInputChanged, diff --git a/ChatApp/app/src/main/kotlin/com/example/chatapp/uicomponents/ChatScreen.kt b/ChatApp/app/src/main/kotlin/com/example/chatapp/uicomponents/ChatScreen.kt index 388dd89..a09910e 100644 --- a/ChatApp/app/src/main/kotlin/com/example/chatapp/uicomponents/ChatScreen.kt +++ b/ChatApp/app/src/main/kotlin/com/example/chatapp/uicomponents/ChatScreen.kt @@ -15,7 +15,9 @@ */ package com.example.chatapp.uicomponents +import androidx.compose.foundation.background import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.PaddingValues import androidx.compose.foundation.layout.consumeWindowInsets @@ -57,6 +59,7 @@ import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.draw.clip +import androidx.compose.ui.graphics.Color import androidx.compose.ui.input.nestedscroll.nestedScroll import androidx.compose.ui.layout.ContentScale import androidx.compose.ui.res.stringResource @@ -90,99 +93,123 @@ fun ChatScreen( val uiState by viewModel.uiState.collectAsStateWithLifecycle() var message by rememberSaveable { mutableStateOf("") } - Scaffold( - modifier = - Modifier - .fillMaxSize() - .nestedScroll(scrollBehavior.nestedScrollConnection), - topBar = { - TopAppBar( - colors = - topAppBarColors( - containerColor = MaterialTheme.colorScheme.primaryContainer, - titleContentColor = MaterialTheme.colorScheme.onPrimaryContainer, - ), - title = { - Text(text = viewModel.recipient.name) - }, - navigationIcon = { - IconButton(onClick = onBackClick) { - Icon( - imageVector = Icons.AutoMirrored.Filled.ArrowBack, - contentDescription = "Back", - ) - } - }, - actions = { - IconButton(onClick = onCallClick) { - Icon( - imageVector = Icons.Default.Call, - contentDescription = "Call", - ) - } - }, + Box(modifier = Modifier.fillMaxSize()) { + uiState.wallpaperPath?.let { path -> + AsyncImage( + model = path, + contentDescription = "Chat Wallpaper", + contentScale = ContentScale.Crop, + modifier = Modifier.fillMaxSize(), ) - }, - bottomBar = { - InputBar( - value = message, - placeholder = stringResource(R.string.input_placeholder), - onInputChanged = { - message = it - }, - onSendClick = { uris -> - viewModel.sendMessage(message, uris) - message = "" - }, - sendEnabled = uiState.botMessageState !is BotMessageState.Generating, + Box( modifier = Modifier - .navigationBarsPadding() - .imePadding(), + .fillMaxSize() + .background(Color.Black.copy(alpha = 0.45f)), ) - }, - ) { innerPadding -> - Column( + } + + Scaffold( modifier = Modifier .fillMaxSize() - .padding(innerPadding) - .consumeWindowInsets(innerPadding), - ) { - MessageList( + .nestedScroll(scrollBehavior.nestedScrollConnection), + containerColor = + if (uiState.wallpaperPath != null) { + Color.Transparent + } else { + MaterialTheme.colorScheme.surfaceVariant + }, + topBar = { + TopAppBar( + colors = + topAppBarColors( + containerColor = MaterialTheme.colorScheme.primaryContainer, + titleContentColor = MaterialTheme.colorScheme.onPrimaryContainer, + ), + title = { + Text(text = viewModel.recipient.name) + }, + navigationIcon = { + IconButton(onClick = onBackClick) { + Icon( + imageVector = Icons.AutoMirrored.Filled.ArrowBack, + contentDescription = "Back", + ) + } + }, + actions = { + IconButton(onClick = onCallClick) { + Icon( + imageVector = Icons.Default.Call, + contentDescription = "Call", + ) + } + }, + ) + }, + bottomBar = { + InputBar( + value = message, + placeholder = stringResource(R.string.input_placeholder), + onInputChanged = { + message = it + }, + onSendClick = { uris -> + viewModel.sendMessage(message, uris) + message = "" + }, + sendEnabled = uiState.botMessageState !is BotMessageState.Generating, + modifier = + Modifier + .navigationBarsPadding() + .imePadding(), + ) + }, + ) { innerPadding -> + Box( modifier = Modifier - .fillMaxWidth() - .padding(horizontal = 16.dp) - .weight(1f), - messages = uiState.messages, - contentPadding = PaddingValues(bottom = 8.dp), - ) - - when (val state = uiState.botMessageState) { - is BotMessageState.Generating -> { - CircularProgressIndicator( + .fillMaxSize() + .padding(innerPadding) + .consumeWindowInsets(innerPadding), + ) { + Column(modifier = Modifier.fillMaxSize()) { + MessageList( modifier = Modifier - .padding(vertical = 8.dp) - .align(Alignment.CenterHorizontally), + .fillMaxWidth() + .padding(horizontal = 16.dp) + .weight(1f), + messages = uiState.messages, + contentPadding = PaddingValues(bottom = 8.dp), ) - } - is BotMessageState.Error -> { - AlertDialog( - onDismissRequest = { viewModel.dismissError() }, - title = { Text(text = stringResource(R.string.error)) }, - text = { Text(text = state.errorMessage) }, - confirmButton = { - Button(onClick = { viewModel.dismissError() }) { - Text(text = stringResource(R.string.dismiss_button)) - } - }, - ) - } + when (val state = uiState.botMessageState) { + is BotMessageState.Generating -> { + CircularProgressIndicator( + modifier = + Modifier + .padding(vertical = 8.dp) + .align(Alignment.CenterHorizontally), + ) + } - else -> { // No additional UI for waiting state + is BotMessageState.Error -> { + AlertDialog( + onDismissRequest = { viewModel.dismissError() }, + title = { Text(text = stringResource(R.string.error)) }, + text = { Text(text = state.errorMessage) }, + confirmButton = { + Button(onClick = { viewModel.dismissError() }) { + Text(text = stringResource(R.string.dismiss_button)) + } + }, + ) + } + + else -> {} + } } } } diff --git a/ChatApp/gradle/gradle-daemon-jvm.properties b/ChatApp/gradle/gradle-daemon-jvm.properties new file mode 100644 index 0000000..6c1139e --- /dev/null +++ b/ChatApp/gradle/gradle-daemon-jvm.properties @@ -0,0 +1,12 @@ +#This file is generated by updateDaemonJvm +toolchainUrl.FREE_BSD.AARCH64=https\://api.foojay.io/disco/v3.0/ids/ec7520a1e057cd116f9544c42142a16b/redirect +toolchainUrl.FREE_BSD.X86_64=https\://api.foojay.io/disco/v3.0/ids/4c4f879899012ff0a8b2e2117df03b0e/redirect +toolchainUrl.LINUX.AARCH64=https\://api.foojay.io/disco/v3.0/ids/ec7520a1e057cd116f9544c42142a16b/redirect +toolchainUrl.LINUX.X86_64=https\://api.foojay.io/disco/v3.0/ids/4c4f879899012ff0a8b2e2117df03b0e/redirect +toolchainUrl.MAC_OS.AARCH64=https\://api.foojay.io/disco/v3.0/ids/73bcfb608d1fde9fb62e462f834a3299/redirect +toolchainUrl.MAC_OS.X86_64=https\://api.foojay.io/disco/v3.0/ids/846ee0d876d26a26f37aa1ce8de73224/redirect +toolchainUrl.UNIX.AARCH64=https\://api.foojay.io/disco/v3.0/ids/ec7520a1e057cd116f9544c42142a16b/redirect +toolchainUrl.UNIX.X86_64=https\://api.foojay.io/disco/v3.0/ids/4c4f879899012ff0a8b2e2117df03b0e/redirect +toolchainUrl.WINDOWS.AARCH64=https\://api.foojay.io/disco/v3.0/ids/9482ddec596298c84656d31d16652665/redirect +toolchainUrl.WINDOWS.X86_64=https\://api.foojay.io/disco/v3.0/ids/39701d92e1756bb2f141eb67cd4c660e/redirect +toolchainVersion=21 diff --git a/ChatApp/shared/build.gradle.kts b/ChatApp/shared/build.gradle.kts index b9aba68..5a720e6 100644 --- a/ChatApp/shared/build.gradle.kts +++ b/ChatApp/shared/build.gradle.kts @@ -48,5 +48,4 @@ dependencies { // App functions implementation(libs.androidx.appfunctions) ksp(libs.androidx.appfunctions.compiler) - } diff --git a/ChatApp/shared/src/main/kotlin/com/example/chatapp/BaseChatApplication.kt b/ChatApp/shared/src/main/kotlin/com/example/chatapp/BaseChatApplication.kt index b051a26..4d53914 100644 --- a/ChatApp/shared/src/main/kotlin/com/example/chatapp/BaseChatApplication.kt +++ b/ChatApp/shared/src/main/kotlin/com/example/chatapp/BaseChatApplication.kt @@ -16,4 +16,5 @@ package com.example.chatapp import android.app.Application + abstract class BaseChatApplication : Application() diff --git a/ChatApp/shared/src/main/kotlin/com/example/chatapp/ChatViewModel.kt b/ChatApp/shared/src/main/kotlin/com/example/chatapp/ChatViewModel.kt index be30019..7524d6f 100644 --- a/ChatApp/shared/src/main/kotlin/com/example/chatapp/ChatViewModel.kt +++ b/ChatApp/shared/src/main/kotlin/com/example/chatapp/ChatViewModel.kt @@ -23,6 +23,7 @@ import com.example.chatapp.data.CallManager import com.example.chatapp.data.DisplayMessage import com.example.chatapp.data.MessageRepository import com.example.chatapp.data.RecipientsRepository +import com.example.chatapp.data.WallpaperRepository import dagger.assisted.Assisted import dagger.assisted.AssistedFactory import dagger.assisted.AssistedInject @@ -64,6 +65,8 @@ data class ChatbotUiState( val messages: List = listOf(), /** The current state of the bot's response generation. */ val botMessageState: BotMessageState = BotMessageState.WaitingForMessage, + /** Optional path to a custom wallpaper image for this chat. */ + val wallpaperPath: String? = null, ) @HiltViewModel(assistedFactory = ChatViewModel.Factory::class) @@ -74,6 +77,7 @@ class ChatViewModel private val messageRepository: MessageRepository, private val callManager: CallManager, private val recipientsRepository: RecipientsRepository, + private val wallpaperRepository: WallpaperRepository, ) : ViewModel() { @AssistedFactory interface Factory { @@ -102,6 +106,11 @@ class ChatViewModel _uiState.update { it.copy(messages = msgs) } } } + viewModelScope.launch { + wallpaperRepository.getWallpaper(recipientId).collect { path -> + _uiState.update { it.copy(wallpaperPath = path) } + } + } } fun startCall() { diff --git a/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/BaseChatAppFunctionService.kt b/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/BaseChatAppFunctionService.kt index caae605..cc13cfd 100644 --- a/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/BaseChatAppFunctionService.kt +++ b/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/BaseChatAppFunctionService.kt @@ -19,19 +19,20 @@ import android.app.PendingIntent import android.content.Intent import android.net.Uri import androidx.annotation.RequiresApi +import androidx.appfunctions.AppFunction import androidx.appfunctions.AppFunctionAppUnknownException import androidx.appfunctions.AppFunctionElementNotFoundException import androidx.appfunctions.AppFunctionInvalidArgumentException import androidx.appfunctions.AppFunctionService import androidx.appfunctions.AppFunctionServiceEntryPoint import androidx.appfunctions.AppFunctionStringValueConstraint -import androidx.appfunctions.AppFunction import com.example.chatapp.data.CallManager import com.example.chatapp.data.MessageRepository import com.example.chatapp.data.RecipientsRepository +import com.example.chatapp.data.WallpaperRepository import dagger.hilt.android.AndroidEntryPoint -import javax.inject.Inject import kotlinx.coroutines.CancellationException +import javax.inject.Inject /** * Service entry point for chat-related AppFunctions such as searching contacts, sending messages, and making calls. @@ -44,9 +45,13 @@ import kotlinx.coroutines.CancellationException ) abstract class BaseChatAppFunctionService : AppFunctionService() { @Inject lateinit var messageRepository: MessageRepository + @Inject lateinit var recipientsRepository: RecipientsRepository + @Inject lateinit var callManager: CallManager + @Inject lateinit var wallpaperRepository: WallpaperRepository + /** * Search for message recipients or chat groups by name or email. * Required workflow: Call this before "send" or "makeCall" to obtain a valid endpointValue (unique ID). @@ -153,9 +158,7 @@ abstract class BaseChatAppFunctionService : AppFunctionService() { * @throws AppFunctionElementNotFoundException If no recipient exists for endpointValue. If thrown, call "searchContacts" to find the correct ID. */ @AppFunction(isDescribedByKDoc = true) - suspend fun makeCall( - endpointValue: String, - ): PendingIntent { + suspend fun makeCall(endpointValue: String): PendingIntent { val recipient = recipientsRepository.getRecipientById(endpointValue) ?: throw AppFunctionElementNotFoundException( @@ -172,4 +175,31 @@ abstract class BaseChatAppFunctionService : AppFunctionService() { PendingIntent.FLAG_ONE_SHOT or PendingIntent.FLAG_IMMUTABLE, ) } + + /** + * Updates the wallpaper image for a specific chat conversation. + * + * @param chatId The unique identifier for the recipient or chat group. + * @param wallpaperUri The URI of the image file to set as the chat wallpaper. + */ + @AppFunction(isDescribedByKDoc = true) + suspend fun updateChatWallpaper( + chatId: String, + wallpaperUri: Uri, + ): Boolean { + val resolvedId = + recipientsRepository.getRecipientById(chatId)?.id + ?: recipientsRepository.getGroupById(chatId)?.id + ?: recipientsRepository.searchAny(chatId, maxCount = 1).firstOrNull()?.endpointValue + ?: chatId + val inputStream = + try { + contentResolver.openInputStream(wallpaperUri) + } catch (e: Exception) { + throw AppFunctionInvalidArgumentException("Cannot open wallpaper stream: ${e.message}") + } ?: throw AppFunctionInvalidArgumentException("Cannot open wallpaper stream") + return inputStream.use { stream -> + wallpaperRepository.setWallpaper(resolvedId, stream) + } + } } diff --git a/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/Util.kt b/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/Util.kt index 81d0354..51ccd17 100644 --- a/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/Util.kt +++ b/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/Util.kt @@ -13,12 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package com.example.chatapp.appfunctions import androidx.appfunctions.AppFunctionSerializable - /** * Represents a result from a contact or group search. */ @@ -67,4 +65,4 @@ data class ChatGroup( val name: String, /** List of members belonging to the group. */ val recipients: List, -) \ No newline at end of file +) diff --git a/ChatApp/shared/src/main/kotlin/com/example/chatapp/data/WallpaperRepository.kt b/ChatApp/shared/src/main/kotlin/com/example/chatapp/data/WallpaperRepository.kt new file mode 100644 index 0000000..e8a85f6 --- /dev/null +++ b/ChatApp/shared/src/main/kotlin/com/example/chatapp/data/WallpaperRepository.kt @@ -0,0 +1,89 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.chatapp.data + +import android.content.Context +import dagger.Binds +import dagger.Module +import dagger.hilt.InstallIn +import dagger.hilt.android.qualifiers.ApplicationContext +import dagger.hilt.components.SingletonComponent +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.flowOn +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.update +import kotlinx.coroutines.withContext +import java.io.File +import java.io.InputStream +import javax.inject.Inject +import javax.inject.Singleton + +interface WallpaperRepository { + fun getWallpaper(chatId: String): Flow + + suspend fun setWallpaper( + chatId: String, + inputStream: InputStream, + ): Boolean +} + +@Singleton +class WallpaperRepositoryImpl + @Inject + constructor( + @ApplicationContext private val context: Context, + ) : WallpaperRepository { + private val wallpapers = MutableStateFlow>(emptyMap()) + + override fun getWallpaper(chatId: String): Flow { + return wallpapers.map { map -> + map[chatId] ?: run { + val dir = File(context.filesDir, "wallpapers") + dir.listFiles { f -> f.name.startsWith("wallpaper_${chatId}_") } + ?.maxByOrNull { it.lastModified() } + ?.absolutePath + } + }.flowOn(Dispatchers.IO) + } + + override suspend fun setWallpaper( + chatId: String, + inputStream: InputStream, + ): Boolean = + withContext(Dispatchers.IO) { + try { + val dir = File(context.filesDir, "wallpapers").apply { mkdirs() } + dir.listFiles { f -> f.name.startsWith("wallpaper_${chatId}_") }?.forEach { it.delete() } + val file = File(dir, "wallpaper_${chatId}_${System.currentTimeMillis()}.jpg") + file.outputStream().use { output -> + inputStream.copyTo(output) + } + wallpapers.update { current -> current + (chatId to file.absolutePath) } + true + } catch (e: Exception) { + false + } + } + } + +@Module +@InstallIn(SingletonComponent::class) +abstract class WallpaperModule { + @Binds + abstract fun bindWallpaperRepository(impl: WallpaperRepositoryImpl): WallpaperRepository +} diff --git a/ChatApp/shared/src/main/kotlin/com/example/chatapp/util/Linkify.kt b/ChatApp/shared/src/main/kotlin/com/example/chatapp/util/Linkify.kt index f0a8f93..453ff4f 100644 --- a/ChatApp/shared/src/main/kotlin/com/example/chatapp/util/Linkify.kt +++ b/ChatApp/shared/src/main/kotlin/com/example/chatapp/util/Linkify.kt @@ -24,7 +24,10 @@ import androidx.compose.ui.text.TextLinkStyles import androidx.compose.ui.text.buildAnnotatedString import androidx.compose.ui.text.style.TextDecoration -fun linkifyString(text: String, linkColor: Color? = null): AnnotatedString { +fun linkifyString( + text: String, + linkColor: Color? = null, +): AnnotatedString { val matcher = Patterns.WEB_URL.matcher(text) var lastIndex = 0 return buildAnnotatedString { @@ -35,22 +38,24 @@ fun linkifyString(text: String, linkColor: Color? = null): AnnotatedString { append(text.substring(lastIndex, start)) - val uriStr = if (url.startsWith("http://", ignoreCase = true) || url.startsWith("https://", ignoreCase = true)) { - url - } else { - "https://$url" - } - - val linkStyle = SpanStyle( - color = linkColor ?: Color.Unspecified, - textDecoration = TextDecoration.Underline - ) + val uriStr = + if (url.startsWith("http://", ignoreCase = true) || url.startsWith("https://", ignoreCase = true)) { + url + } else { + "https://$url" + } + + val linkStyle = + SpanStyle( + color = linkColor ?: Color.Unspecified, + textDecoration = TextDecoration.Underline, + ) pushLink( LinkAnnotation.Url( url = uriStr, - styles = TextLinkStyles(style = linkStyle) - ) + styles = TextLinkStyles(style = linkStyle), + ), ) append(url) pop() diff --git a/agent/app/build.gradle.kts b/agent/app/build.gradle.kts index b2a3b3d..ec6f854 100644 --- a/agent/app/build.gradle.kts +++ b/agent/app/build.gradle.kts @@ -16,6 +16,7 @@ plugins { alias(libs.plugins.android.application) alias(libs.plugins.kotlin.compose) + alias(libs.plugins.kotlin.serialization) alias(libs.plugins.ksp) alias(libs.plugins.hilt) alias(libs.plugins.screenshot) @@ -60,7 +61,10 @@ android { dimension = "mode" buildConfigField("Boolean", "IS_RETAIL", "true") - val containsRetail = gradle.startParameter.taskNames.any { it.contains("Retail", ignoreCase = true) } + val containsRetail = + gradle.startParameter.taskNames.any { + it.contains("Retail", ignoreCase = true) + } val apiKey = project.findProperty("GEMINI_API_KEY") as? String ?: "" if (containsRetail && apiKey.isEmpty()) { throw GradleException( diff --git a/agent/app/src/main/AndroidManifest.xml b/agent/app/src/main/AndroidManifest.xml index 7fa52ed..699a2e4 100644 --- a/agent/app/src/main/AndroidManifest.xml +++ b/agent/app/src/main/AndroidManifest.xml @@ -59,6 +59,16 @@ android:exported="false" tools:replace="android:label,android:theme" /> + + + + + geocoder.getFromLocationName( + address, + 1, + object : Geocoder.GeocodeListener { + override fun onGeocode(addresses: MutableList
) { + val location = addresses.firstOrNull() + if (location != null) { + continuation.resume( + LatLng(location.latitude, location.longitude), + ) + } else { + continuation.resume(null) + } + } + + override fun onError(errorMessage: String?) { + continuation.resume(null) + } + }, + ) + } + } catch (e: Exception) { + throw IllegalStateException(e.message, e) + } + } + } + + /** + * Retrieve the current latitude and longitude coordinates of the device. + * + * @return The current location coordinates of the device, or null if location is unavailable or + * permission is denied. + */ + @SuppressLint("MissingPermission") + @AppFunction(isDescribedByKDoc = true) + suspend fun getCurrentLocation(): LatLng? = + withContext(Dispatchers.Default) { + // Check permissions + val hasFineLocation = + ContextCompat.checkSelfPermission( + this@BaseBuiltInAppFunctionService, + Manifest.permission.ACCESS_FINE_LOCATION, + ) == PackageManager.PERMISSION_GRANTED + + val hasCoarseLocation = + ContextCompat.checkSelfPermission( + this@BaseBuiltInAppFunctionService, + Manifest.permission.ACCESS_COARSE_LOCATION, + ) == PackageManager.PERMISSION_GRANTED + + if (!hasFineLocation && !hasCoarseLocation) { + throw IllegalStateException("Location permission is not granted") + } + + val locationManager = + this@BaseBuiltInAppFunctionService.getSystemService(Context.LOCATION_SERVICE) as LocationManager + + try { + // Try GPS Provider first + var location = + if (locationManager.isProviderEnabled(LocationManager.GPS_PROVIDER)) { + locationManager.getLastKnownLocation(LocationManager.GPS_PROVIDER) + } else { + null + } + + // Fallback to Network Provider if GPS is not available + if (location == null && + locationManager.isProviderEnabled(LocationManager.NETWORK_PROVIDER) + ) { + location = + locationManager.getLastKnownLocation(LocationManager.NETWORK_PROVIDER) + } + + if (location != null) { + LatLng(location.latitude, location.longitude) + } else { + null + } + } catch (e: Exception) { + throw IllegalStateException(e.message, e) + } + } + + /** Represents the latitude and longitude coordinates. */ + @AppFunctionSerializable(isDescribedByKDoc = true) + data class LatLng( + /** The latitude coordinate. */ + val latitude: Double, + /** The longitude coordinate. */ + val longitude: Double, + ) + + /** + * Generates an image from a text prompt and returns the remote image URI. + * + * @param prompt The text prompt describing the image to generate (e.g., "futuristic cityscape at sunset"). + * @param aspectRatio Optional aspect ratio for the image (e.g., "16:9", "1:1"). + * @return A GeneratedImageResult containing the generated remote image URI. + */ + @AppFunction(isDescribedByKDoc = true) + suspend fun generateImage( + prompt: String, + aspectRatio: String? = null, + ): GeneratedImageResult = + withContext(Dispatchers.IO) { + val apiKey = getOrFetchApiKey() + val requestPayload = buildImageGenerationPayload(prompt, aspectRatio) + val responseText = executeImageRequest(apiKey, requestPayload) + saveBase64ImageToCache(responseText, prompt) + } + + private suspend fun getOrFetchApiKey(): String { + val apiKey = + settingsDataStore.data + .first()[stringPreferencesKey("gemini_api_key")] + ?.takeIf { it.isNotBlank() } + ?: BuildConfig.GEMINI_API_KEY.takeIf { it.isNotBlank() } + if (apiKey.isNullOrBlank()) { + throw IllegalStateException( + "Gemini API key is not configured. Please set gemini_api_key in settings.", + ) + } + return apiKey + } + + private fun buildImageGenerationPayload( + prompt: String, + aspectRatio: String?, + ): String = + JSONObject().apply { + put( + "contents", + JSONArray().apply { + put( + JSONObject().apply { + put( + "parts", + JSONArray().apply { + put(JSONObject().apply { put("text", prompt) }) + }, + ) + }, + ) + }, + ) + put( + "generationConfig", + JSONObject().apply { + put("responseModalities", JSONArray().apply { put("IMAGE") }) + if (!aspectRatio.isNullOrBlank()) { + put( + "imageConfig", + JSONObject().apply { + put("aspectRatio", aspectRatio) + }, + ) + } + }, + ) + }.toString() + + private fun executeImageRequest( + apiKey: String, + requestPayload: String, + ): String { + val endpointUrl = + "https://generativelanguage.googleapis.com/v1beta/models/gemini-3.1-flash-image-preview:generateContent?key=$apiKey" + val connection = + (URL(endpointUrl).openConnection() as HttpURLConnection).apply { + requestMethod = "POST" + setRequestProperty("Content-Type", "application/json") + doOutput = true + connectTimeout = 30000 + readTimeout = 60000 + } + + return try { + connection.outputStream.use { os -> + os.write(requestPayload.toByteArray(Charsets.UTF_8)) + } + + val responseCode = connection.responseCode + if (responseCode != HttpURLConnection.HTTP_OK) { + val errorBody = + connection.errorStream?.bufferedReader()?.use { it.readText() } + ?: "HTTP $responseCode" + throw IllegalStateException( + "Image generation failed ($responseCode): $errorBody", + ) + } + + connection.inputStream.bufferedReader().use { it.readText() } + } finally { + connection.disconnect() + } + } + + private data class ImageInlineData(val base64: String, val mimeType: String) + + private fun saveBase64ImageToCache( + responseText: String, + prompt: String, + ): GeneratedImageResult { + val responseJson = JSONObject(responseText) + val candidates = responseJson.optJSONArray("candidates") + if (candidates == null || candidates.length() == 0) { + throw IllegalStateException( + "No candidates returned from Gemini image generation API", + ) + } + + val parts = + candidates + .getJSONObject(0) + .optJSONObject("content") + ?.optJSONArray("parts") + if (parts == null || parts.length() == 0) { + throw IllegalStateException( + "No parts returned in candidate content. Gemini response: $responseText", + ) + } + + val inlineResult = + (0 until parts.length()) + .asSequence() + .mapNotNull { i -> + val part = parts.getJSONObject(i) + val inlineData = + part.optJSONObject("inlineData") + ?: part.optJSONObject("inline_data") + if (inlineData != null) { + val data = inlineData.optString("data") + val mime = + inlineData + .optString("mimeType") + .takeIf { it.isNotBlank() } + ?: inlineData.optString("mime_type").takeIf { it.isNotBlank() } + ?: "image/png" + if (data.isNotBlank()) ImageInlineData(data, mime) else null + } else { + null + } + } + .firstOrNull() + ?: throw IllegalStateException( + "No inlineData image found in response parts. Gemini response: $responseText", + ) + + val imageBytes = Base64.decode(inlineResult.base64, Base64.DEFAULT) + val extension = + when { + inlineResult.mimeType.contains("jpeg", ignoreCase = true) || + inlineResult.mimeType.contains("jpg", ignoreCase = true) -> "jpg" + else -> "png" + } + val cachedFile = + File( + cacheDir, + "generated_${UUID.randomUUID()}.$extension", + ) + cachedFile.writeBytes(imageBytes) + + val authority = "$packageName.fileprovider" + val contentUri = FileProvider.getUriForFile(this, authority, cachedFile) + return GeneratedImageResult( + imageUri = contentUri.toString(), + mimeType = inlineResult.mimeType, + prompt = prompt, + ) + } + + /** Represents the result of an image generation request. */ + @AppFunctionSerializable + data class GeneratedImageResult( + /** The remote URI or URL of the generated image. */ + val imageUri: String, + /** The MIME type of the generated image. */ + val mimeType: String, + /** The original prompt used to generate the image. */ + val prompt: String, + ) +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/FakeAppFunctions.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/BaseFakeAppFunctionService.kt similarity index 96% rename from agent/app/src/main/java/com/example/appfunctions/agent/data/FakeAppFunctions.kt rename to agent/app/src/main/java/com/example/appfunctions/agent/data/BaseFakeAppFunctionService.kt index 1f82650..2048ee9 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/FakeAppFunctions.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/BaseFakeAppFunctionService.kt @@ -37,9 +37,7 @@ abstract class BaseFakeAppFunctionService : AppFunctionService() { * @return The test response containing output data. */ @AppFunction(isDescribedByKDoc = true) - suspend fun fakeFunction( - params: FakeParams, - ): FakeResponse { + suspend fun fakeFunction(params: FakeParams): FakeResponse { return FakeResponse("success") } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/BuiltInAppFunctions.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/BuiltInAppFunctions.kt deleted file mode 100644 index 693f7bb..0000000 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/BuiltInAppFunctions.kt +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright 2026 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.example.appfunctions.agent.data - -import android.Manifest -import android.annotation.SuppressLint -import android.content.Context -import android.content.pm.PackageManager -import android.location.Address -import android.location.Geocoder -import android.location.LocationManager -import androidx.annotation.RequiresApi -import androidx.appfunctions.AppFunction -import androidx.appfunctions.AppFunctionSerializable -import androidx.appfunctions.AppFunctionService -import androidx.appfunctions.AppFunctionServiceEntryPoint -import androidx.core.content.ContextCompat -import dagger.hilt.android.AndroidEntryPoint -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext -import kotlin.coroutines.resume -import kotlin.coroutines.suspendCoroutine - -/** Built-in AppFunctions for location and geocoding services. */ -@RequiresApi(36) -@AndroidEntryPoint -@AppFunctionServiceEntryPoint( - serviceName = "BuiltInAppFunctionService", - appFunctionXmlFileName = "builtin_app_function_service", -) -abstract class BaseBuiltInAppFunctionService : AppFunctionService() { - /** - * Geocode a physical address string into its latitude and longitude coordinates. - * - * @param address The physical address to geocode (e.g., "1600 Amphitheatre Pkwy, Mountain View, - * CA"). - * @return The latitude and longitude coordinates of the address, or null if geocoding fails. - */ - @AppFunction(isDescribedByKDoc = true) - suspend fun geocodeAddress( - address: String, - ): LatLng? { - if (!Geocoder.isPresent()) { - return null - } - - val geocoder = Geocoder(this) - - return withContext(Dispatchers.IO) { - try { - suspendCoroutine { continuation -> - geocoder.getFromLocationName( - address, - 1, - object : Geocoder.GeocodeListener { - override fun onGeocode(addresses: MutableList
) { - val location = addresses.firstOrNull() - if (location != null) { - continuation.resume( - LatLng(location.latitude, location.longitude), - ) - } else { - continuation.resume(null) - } - } - - override fun onError(errorMessage: String?) { - continuation.resume(null) - } - }, - ) - } - } catch (e: Exception) { - throw IllegalStateException(e.message, e) - } - } - } - - /** - * Retrieve the current latitude and longitude coordinates of the device. - * - * @return The current location coordinates of the device, or null if location is unavailable or - * permission is denied. - */ - @SuppressLint("MissingPermission") - @AppFunction(isDescribedByKDoc = true) - suspend fun getCurrentLocation(): LatLng? = - withContext(Dispatchers.Default) { - // Check permissions - val hasFineLocation = - ContextCompat.checkSelfPermission( - this@BaseBuiltInAppFunctionService, - Manifest.permission.ACCESS_FINE_LOCATION, - ) == PackageManager.PERMISSION_GRANTED - - val hasCoarseLocation = - ContextCompat.checkSelfPermission( - this@BaseBuiltInAppFunctionService, - Manifest.permission.ACCESS_COARSE_LOCATION, - ) == PackageManager.PERMISSION_GRANTED - - if (!hasFineLocation && !hasCoarseLocation) { - throw IllegalStateException("Location permission is not granted") - } - - val locationManager = - this@BaseBuiltInAppFunctionService.getSystemService(Context.LOCATION_SERVICE) as LocationManager - - try { - // Try GPS Provider first - var location = - if (locationManager.isProviderEnabled(LocationManager.GPS_PROVIDER)) { - locationManager.getLastKnownLocation(LocationManager.GPS_PROVIDER) - } else { - null - } - - // Fallback to Network Provider if GPS is not available - if (location == null && - locationManager.isProviderEnabled(LocationManager.NETWORK_PROVIDER) - ) { - location = - locationManager.getLastKnownLocation(LocationManager.NETWORK_PROVIDER) - } - - if (location != null) { - LatLng(location.latitude, location.longitude) - } else { - null - } - } catch (e: Exception) { - throw IllegalStateException(e.message, e) - } - } - - /** Represents the latitude and longitude coordinates. */ - @AppFunctionSerializable(isDescribedByKDoc = true) - data class LatLng( - /** The latitude coordinate. */ - val latitude: Double, - /** The longitude coordinate. */ - val longitude: Double, - ) -} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt index 7e2bd18..4627e76 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt @@ -57,8 +57,10 @@ class GeminiProviderImpl apiKey: String, modelName: String, ): LlmResponse { + val sortedTools = tools.sortedByDescending { it.id.startsWith(it.packageName) } + val uniqueTools = sortedTools.distinctBy { toolConverter.getToolName(it) } val convertedTools = - tools.mapNotNull { tool -> + uniqueTools.mapNotNull { tool -> try { buildJsonObject { put(KEY_TYPE, JsonPrimitive(VALUE_FUNCTION)) @@ -187,7 +189,9 @@ class GeminiProviderImpl } val callId = stepObj["id"]?.jsonPrimitive?.content - ?: return LlmResponse.Error("Function call missing call_id in response") + ?: return LlmResponse.Error( + "Function call missing call_id in response", + ) parts.add( LlmResponsePart.ToolCall( packageName = packageName, @@ -223,7 +227,12 @@ class GeminiProviderImpl private fun getSystemInstruction(): String { val currentDate = LocalDate.now().toString() - return "You are an assistant running on Android. Be concise, direct and helpful. Today's date is $currentDate." + return """ + You are an AI assistant running on Android. Today's date is $currentDate. + Always reply to the user using concise, friendly, natural human language. Never respond to the user with raw JSON or structured code blocks unless explicitly asked to write code. + When a user asks you to generate an image, call the generateImage tool. After generateImage completes, confirm to the user in a natural sentence (for example: "I have generated that image for you."). + When a user asks you to generate an image and use it in an app (for example, setting a chat wallpaper or attaching an image to a note), first call generateImage, then call the target app function passing the returned imageUri. + """.trimIndent() } companion object { diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt index 7de144e..cc10981 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt @@ -67,12 +67,16 @@ class GeminiToolConverter parameter.dataType, tool.components, tool.id, + parameterName = parameter.name, ) put( parameter.name, buildJsonObject { typeSchema.forEach { (key, value) -> put(key, value) } - put(KEY_DESCRIPTION, JsonPrimitive(parameter.description)) + put( + KEY_DESCRIPTION, + JsonPrimitive(parameter.description), + ) }, ) } @@ -82,7 +86,13 @@ class GeminiToolConverter if (requiredParams.isNotEmpty()) { put( KEY_REQUIRED, - buildJsonArray { requiredParams.forEach { add(JsonPrimitive(it)) } }, + buildJsonArray { + requiredParams.forEach { + add( + JsonPrimitive(it), + ) + } + }, ) } }, @@ -118,11 +128,15 @@ class GeminiToolConverter components: AppFunctionComponentsMetadata, functionId: String, visitedReferences: Set = emptySet(), + parameterName: String? = null, ): JsonObject { return when (dataType) { is AppFunctionStringTypeMetadata -> buildJsonObject { put(KEY_TYPE, JsonPrimitive(VALUE_STRING)) + if (isFileReferenceParameter(parameterName)) { + put(KEY_FORMAT, JsonPrimitive(VALUE_FILE_REFERENCE)) + } val enumValues = dataType.enumValues if (!enumValues.isNullOrEmpty()) { put( @@ -168,39 +182,58 @@ class GeminiToolConverter components, functionId, visitedReferences, + parameterName, ), ) } is AppFunctionObjectTypeMetadata -> - buildJsonObject { - put(KEY_TYPE, JsonPrimitive(VALUE_OBJECT)) - put( - KEY_PROPERTIES, - buildJsonObject { - dataType.properties.forEach { (name, type) -> - put( - name, - mapDataTypeToGeminiSchema( - type, - components, - functionId, - visitedReferences, - ), - ) - } - }, - ) - if (dataType.required.isNotEmpty()) { + if (dataType.qualifiedName == "android.net.Uri") { + buildJsonObject { + put(KEY_TYPE, JsonPrimitive(VALUE_STRING)) + put(KEY_FORMAT, JsonPrimitive(VALUE_FILE_REFERENCE)) + } + } else { + buildJsonObject { + put(KEY_TYPE, JsonPrimitive(VALUE_OBJECT)) put( - KEY_REQUIRED, - buildJsonArray { - dataType.required.forEach { name -> add(JsonPrimitive(name)) } + KEY_PROPERTIES, + buildJsonObject { + dataType.properties.forEach { (name, type) -> + put( + name, + mapDataTypeToGeminiSchema( + type, + components, + functionId, + visitedReferences, + parameterName = name, + ), + ) + } }, ) + if (dataType.required.isNotEmpty()) { + put( + KEY_REQUIRED, + buildJsonArray { + dataType.required.forEach { name -> + add( + JsonPrimitive(name), + ) + } + }, + ) + } } } is AppFunctionReferenceTypeMetadata -> { val referenceKey = dataType.referenceDataType + if (referenceKey == "android.net.Uri") { + return buildJsonObject { + put(KEY_TYPE, JsonPrimitive(VALUE_STRING)) + put(KEY_FORMAT, JsonPrimitive(VALUE_FILE_REFERENCE)) + } + } if (visitedReferences.contains(referenceKey)) { Log.d( "GeminiToolConverter", @@ -218,6 +251,7 @@ class GeminiToolConverter components, functionId, visitedReferences + referenceKey, + parameterName, ) } else -> @@ -227,12 +261,30 @@ class GeminiToolConverter } } + private fun isFileReferenceParameter(parameterName: String?): Boolean { + if (parameterName == null) return false + if (parameterName in KNOWN_FILE_REFERENCE_PARAM_NAMES) return true + return parameterName.endsWith("Uri", ignoreCase = true) || + parameterName.endsWith("Uris", ignoreCase = true) + } + companion object { private const val TOOL_ID_SEPARATOR = "_" private const val KEY_NAME = "name" private const val KEY_DESCRIPTION = "description" private const val KEY_PARAMETERS = "parameters" private const val KEY_TYPE = "type" + private const val KEY_FORMAT = "format" + private const val VALUE_FILE_REFERENCE = "file_reference" + private val KNOWN_FILE_REFERENCE_PARAM_NAMES = + setOf( + "wallpaperUri", + "imageUri", + "attachmentUri", + "ringtoneUri", + "profilePictureUri", + "audioUri", + ) private const val VALUE_OBJECT = "object" private const val KEY_PROPERTIES = "properties" private const val KEY_REQUIRED = "required" diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/db/AppDatabase.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/db/AppDatabase.kt index ae383e6..46d31c1 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/db/AppDatabase.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/db/AppDatabase.kt @@ -17,11 +17,14 @@ package com.example.appfunctions.agent.data.db import androidx.room.Database import androidx.room.RoomDatabase +import androidx.room.TypeConverters import com.example.appfunctions.agent.data.db.dao.ChatDao +import com.example.appfunctions.agent.data.db.entities.MessageAttachmentConverter import com.example.appfunctions.agent.data.db.entities.MessageEntity import com.example.appfunctions.agent.data.db.entities.ThreadEntity -@Database(entities = [ThreadEntity::class, MessageEntity::class], version = 2, exportSchema = false) +@Database(entities = [ThreadEntity::class, MessageEntity::class], version = 3, exportSchema = false) +@TypeConverters(MessageAttachmentConverter::class) abstract class AppDatabase : RoomDatabase() { abstract fun chatDao(): ChatDao } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/db/entities/MessageEntity.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/db/entities/MessageEntity.kt index 075700b..32022ba 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/db/entities/MessageEntity.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/db/entities/MessageEntity.kt @@ -19,6 +19,9 @@ import androidx.room.Entity import androidx.room.ForeignKey import androidx.room.Index import androidx.room.PrimaryKey +import kotlinx.serialization.Serializable +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json @Entity( tableName = "messages", @@ -46,8 +49,35 @@ data class MessageEntity( */ val pendingIntentId: String? = null, val targetPackageName: String? = null, + val attachments: List = emptyList(), ) +@Serializable +data class MessageAttachment( + val uri: String, + val mimeType: String, +) + +class MessageAttachmentConverter { + private val dbJson = + Json { + ignoreUnknownKeys = true + encodeDefaults = true + } + + @androidx.room.TypeConverter + fun fromAttachments(attachments: List): String = dbJson.encodeToString(attachments) + + @androidx.room.TypeConverter + fun toAttachments(jsonString: String?): List = + if (jsonString.isNullOrBlank()) { + emptyList() + } else { + runCatching { dbJson.decodeFromString>(jsonString) } + .getOrDefault(emptyList()) + } +} + enum class MessageRole { USER, ASSISTANT, diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt b/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt index 78ddcc7..3757f9d 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt @@ -42,7 +42,7 @@ import io.ktor.serialization.kotlinx.json.json import kotlinx.serialization.json.Json import javax.inject.Singleton -private val Context.settingsDataStore: DataStore by +val Context.settingsDataStore: DataStore by preferencesDataStore(name = "settings") @Module diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt index 6b47c36..aa99700 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt @@ -16,11 +16,20 @@ package com.example.appfunctions.agent.domain import android.app.PendingIntent +import android.content.Context +import android.content.Intent +import android.net.Uri import android.util.Log -import androidx.appfunctions.AppFunctionException +import androidx.appfunctions.metadata.AppFunctionArrayTypeMetadata +import androidx.appfunctions.metadata.AppFunctionDataTypeMetadata import androidx.appfunctions.metadata.AppFunctionMetadata +import androidx.appfunctions.metadata.AppFunctionObjectTypeMetadata +import androidx.appfunctions.metadata.AppFunctionParameterMetadata +import androidx.appfunctions.metadata.AppFunctionReferenceTypeMetadata +import androidx.core.content.FileProvider import com.example.appfunctions.agent.data.LlmProviderName import com.example.appfunctions.agent.data.SettingsRepository +import com.example.appfunctions.agent.data.db.entities.MessageAttachment import com.example.appfunctions.agent.data.db.entities.MessageEntity import com.example.appfunctions.agent.data.db.entities.MessageProcessingStatus import com.example.appfunctions.agent.data.db.entities.MessageRole @@ -38,7 +47,7 @@ import com.example.appfunctions.agent.domain.chat.UpdateMessageUseCase import com.example.appfunctions.agent.domain.chat.UpdateThreadParams import com.example.appfunctions.agent.domain.chat.UpdateThreadUseCase import com.example.appfunctions.agent.domain.pendingintent.SavePendingIntentUseCase -import java.util.UUID +import dagger.hilt.android.qualifiers.ApplicationContext import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.coroutineScope @@ -50,6 +59,11 @@ import kotlinx.coroutines.flow.filterNotNull import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.withContext +import org.json.JSONObject +import java.io.File +import java.net.HttpURLConnection +import java.net.URL +import java.util.UUID import javax.inject.Inject import javax.inject.Singleton @@ -58,6 +72,7 @@ import javax.inject.Singleton class AgentOrchestrator @Inject constructor( + @ApplicationContext private val context: Context, private val manageThreadsUseCase: ManageThreadsUseCase, private val observePendingMessagesUseCase: ObservePendingMessagesUseCase, private val sendMessageUseCase: SendMessageUseCase, @@ -82,7 +97,9 @@ class AgentOrchestrator suspend fun observeAndProcessMessages(threadId: String) = coroutineScope { val threadStateFlow = - manageThreadsUseCase.getThread(threadId).stateIn(this, SharingStarted.Eagerly, null) + manageThreadsUseCase.getThread( + threadId, + ).stateIn(this, SharingStarted.Eagerly, null) observePendingMessagesUseCase(threadId).collect { message -> if (message != null) { @@ -144,11 +161,14 @@ class AgentOrchestrator disconnectedApps: Set, targetPackageName: String?, ): List { - return allTools.filter { metadata -> - metadata.isEnabled && - metadata.packageName !in disconnectedApps && - (targetPackageName == null || metadata.packageName == targetPackageName) - } + return allTools + .filter { metadata -> + metadata.isEnabled && + metadata.packageName !in disconnectedApps && + (targetPackageName == null || metadata.packageName == targetPackageName) + } + .sortedByDescending { metadata -> metadata.id.startsWith(metadata.packageName) } + .distinctBy { metadata -> metadata.id } } private suspend fun runInteractionLoop( @@ -166,6 +186,7 @@ class AgentOrchestrator var currentToolOutputs = emptyList() var continueLoop = true var currentInput = initialInput + val capturedAttachments = mutableListOf() while (continueLoop) { val llmInput = prepareLlmInput(currentToolOutputs, currentInput) @@ -180,7 +201,10 @@ class AgentOrchestrator modelName = modelName, ) - when (val handleResult = handleLlmResponse(response, message, tools)) { + when ( + val handleResult = + handleLlmResponse(response, message, tools, capturedAttachments) + ) { is HandleResult.Continue -> { currentToolOutputs = handleResult.toolOutputs previousInteractionId = handleResult.interactionId @@ -232,6 +256,7 @@ class AgentOrchestrator response: LlmResponse, message: MessageEntity, tools: List, + capturedAttachments: MutableList, ): HandleResult { return when (response) { is LlmResponse.Success -> { @@ -252,6 +277,21 @@ class AgentOrchestrator if (toolCalls.isNotEmpty()) { when (val toolResult = executeToolCalls(toolCalls, tools, message)) { is ExecuteToolCallsResult.Success -> { + for (output in toolResult.toolOutputs) { + runCatching { + val json = JSONObject(output.result) + val uri = json.optString("imageUri") + if (uri.isNotBlank()) { + val mimeType = + json.optString("mimeType") + .takeIf { it.isNotBlank() } + ?: "image/png" + capturedAttachments.add( + MessageAttachment(uri = uri, mimeType = mimeType), + ) + } + } + } if (textContent.isNotEmpty()) { sendMessageUseCase( threadId = message.threadId, @@ -260,7 +300,10 @@ class AgentOrchestrator processingStatus = MessageProcessingStatus.PROCESSED, ) } - HandleResult.Continue(toolResult.toolOutputs, response.interactionId) + HandleResult.Continue( + toolResult.toolOutputs, + response.interactionId, + ) } is ExecuteToolCallsResult.PendingIntentAction -> { @@ -283,12 +326,13 @@ class AgentOrchestrator } } } else { - if (textContent.isNotEmpty()) { + if (textContent.isNotEmpty() || capturedAttachments.isNotEmpty()) { sendMessageUseCase( threadId = message.threadId, role = MessageRole.ASSISTANT, textContent = textContent, processingStatus = MessageProcessingStatus.PROCESSED, + attachments = capturedAttachments, ) } HandleResult.Stop @@ -297,7 +341,11 @@ class AgentOrchestrator is LlmResponse.Error -> { Log.e("AgentOrchestrator", "LLM Error: ${response.errorMessage}") - completeMessageWithError(message.messageId, message.threadId, response.errorMessage) + completeMessageWithError( + message.messageId, + message.threadId, + response.errorMessage, + ) _status.value = AgentStatus.Idle HandleResult.Stop } @@ -327,7 +375,26 @@ class AgentOrchestrator return ExecuteToolCallsResult.Error } - val convertedInputs = toolCall.arguments.filterValues { it != null } as Map + val rawConvertedInputs = toolCall.arguments.filterValues { it != null } as Map + val convertedInputs = + try { + withContext(Dispatchers.IO) { + resolveRemoteFileReferencesRecursively( + context = context, + parametersMetadata = matchingTool.parameters, + inputs = rawConvertedInputs, + ) + } + } catch (e: Exception) { + completeMessageWithError( + message.messageId, + message.threadId, + "Failed to download remote file reference: ${e.message}", + ) + return ExecuteToolCallsResult.Error + } + + grantContentUriPermissionsRecursively(context, toolCall.packageName, convertedInputs) val appFunctionDataResult = withContext(Dispatchers.Default) { @@ -366,7 +433,7 @@ class AgentOrchestrator } is ExecuteAppFunctionResult.PendingIntentAction -> { - val pendingIntentId = UUID.randomUUID().toString() + val pendingIntentId = java.util.UUID.randomUUID().toString() return ExecuteToolCallsResult.PendingIntentAction( pendingIntentId, executionResult.pendingIntent, @@ -378,7 +445,8 @@ class AgentOrchestrator if (exception is CancellationException) { throw exception } - val appFunctionException = AppFunctionExceptionFormatter.getAppFunctionException(exception) + val appFunctionException = + AppFunctionExceptionFormatter.getAppFunctionException(exception) if (appFunctionException != null) { results.add( ToolOutput( @@ -419,4 +487,162 @@ class AgentOrchestrator processingStatus = MessageProcessingStatus.FAILED, ) } + + private fun resolveRemoteFileReferencesRecursively( + context: Context, + parametersMetadata: List, + inputs: Map, + ): Map { + val paramMap = parametersMetadata.associateBy { it.name } + return inputs.mapValues { (key, value) -> + val paramMeta = paramMap[key] + resolveValueRecursively(context, paramMeta?.dataType, value, key) + } + } + + private fun resolveValueRecursively( + context: Context, + dataType: AppFunctionDataTypeMetadata?, + value: Any, + paramName: String?, + ): Any { + return when (value) { + is String -> { + val shouldResolve = + isFileReferenceParameter(paramName) || isUriMetadata(dataType) + if (shouldResolve && ( + value.startsWith( + "http://", + ) || value.startsWith("https://") + ) + ) { + downloadRemoteFileToContentUri(context, value) + } else { + value + } + } + is Map<*, *> -> { + value.entries.associate { entry -> + val k = entry.key as String + val propType = + (dataType as? AppFunctionObjectTypeMetadata)?.properties?.get(k) + k to ( + entry.value?.let { + resolveValueRecursively( + context, + propType, + it, + k, + ) + } ?: "" + ) + } + } + is List<*> -> { + val itemType = (dataType as? AppFunctionArrayTypeMetadata)?.itemType + value.mapNotNull { item -> + if (item != null) resolveValueRecursively(context, itemType, item, paramName) else null + } + } + else -> value + } + } + + private fun downloadRemoteFileToContentUri( + context: Context, + urlString: String, + ): String { + val url = URL(urlString) + val connection = + (url.openConnection() as HttpURLConnection).apply { + connectTimeout = 10000 + readTimeout = 15000 + } + try { + connection.connect() + val contentType = connection.contentType ?: "" + val ext = + when { + contentType.contains("png", ignoreCase = true) -> "png" + contentType.contains("jpeg", ignoreCase = true) || + contentType.contains("jpg", ignoreCase = true) -> "jpg" + contentType.contains("gif", ignoreCase = true) -> "gif" + contentType.contains("webp", ignoreCase = true) -> "webp" + urlString.substringAfterLast("/", "").contains(".") -> + urlString.substringAfterLast("/").substringAfterLast(".") + else -> "jpg" + } + val cacheDir = File(context.cacheDir, "file_references").apply { mkdirs() } + val file = File(cacheDir, "generated_${UUID.randomUUID()}.$ext") + connection.inputStream.use { input -> + file.outputStream().use { output -> + input.copyTo(output) + } + } + val contentUri = + FileProvider.getUriForFile( + context, + "${context.packageName}.fileprovider", + file, + ) + return contentUri.toString() + } finally { + connection.disconnect() + } + } + + private fun grantContentUriPermissionsRecursively( + context: Context, + targetPackageName: String, + value: Any?, + ) { + when (value) { + is String -> { + if (value.startsWith("content://")) { + runCatching { + val uri = Uri.parse(value) + context.grantUriPermission( + targetPackageName, + uri, + Intent.FLAG_GRANT_READ_URI_PERMISSION, + ) + } + } + } + is Map<*, *> -> + value.values.forEach { + grantContentUriPermissionsRecursively(context, targetPackageName, it) + } + is List<*> -> + value.forEach { + grantContentUriPermissionsRecursively(context, targetPackageName, it) + } + } + } + + private fun isFileReferenceParameter(parameterName: String?): Boolean { + if (parameterName == null) return false + if (parameterName in KNOWN_FILE_REFERENCE_PARAM_NAMES) return true + return parameterName.endsWith("Uri", ignoreCase = true) || + parameterName.endsWith("Uris", ignoreCase = true) + } + + private fun isUriMetadata(dataType: AppFunctionDataTypeMetadata?): Boolean { + if (dataType == null) return false + if (dataType is AppFunctionObjectTypeMetadata && dataType.qualifiedName == "android.net.Uri") return true + if (dataType is AppFunctionReferenceTypeMetadata && dataType.referenceDataType == "android.net.Uri") return true + return false + } + + companion object { + private val KNOWN_FILE_REFERENCE_PARAM_NAMES = + setOf( + "wallpaperUri", + "imageUri", + "attachmentUri", + "ringtoneUri", + "profilePictureUri", + "audioUri", + ) + } } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/AppFunctionExceptionFormatter.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/AppFunctionExceptionFormatter.kt index c3b381a..fc42248 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/AppFunctionExceptionFormatter.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/AppFunctionExceptionFormatter.kt @@ -22,7 +22,10 @@ object AppFunctionExceptionFormatter { /** * Formats the given [exception] including its class name and message. */ - fun format(exception: AppFunctionException, functionId: String? = null): String { + fun format( + exception: AppFunctionException, + functionId: String? = null, + ): String { val className = exception.javaClass.simpleName val message = exception.errorMessage ?: exception.message ?: "No error message provided" val prefix = if (functionId != null) "Tool execution failed for $functionId: " else "" diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCase.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCase.kt index 7fa27e7..799331a 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCase.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCase.kt @@ -98,8 +98,17 @@ class ConvertInputToAppFunctionDataUseCase } } is AppFunctionObjectTypeMetadata -> { - val objData = convertObject(dataType, value as Map, components) - builder.setAppFunctionData(name, objData) + if (dataType.qualifiedName == "android.net.Uri" && value is String) { + val uriPropertyName = dataType.properties.keys.firstOrNull() ?: "uri" + val uriData = + AppFunctionData.Builder(dataType, components) + .setString(uriPropertyName, value) + .build() + builder.setAppFunctionData(name, uriData) + } else { + val objData = convertObject(dataType, value as Map, components) + builder.setAppFunctionData(name, objData) + } } is AppFunctionArrayTypeMetadata -> { setArrayValue(builder, name, dataType, value as List, components) @@ -109,8 +118,17 @@ class ConvertInputToAppFunctionDataUseCase val objectType = components.dataTypes[referenceKey] as? AppFunctionObjectTypeMetadata if (objectType != null) { - val objData = convertObject(objectType, value as Map, components) - builder.setAppFunctionData(name, objData) + if (referenceKey == "android.net.Uri" && value is String) { + val uriPropertyName = objectType.properties.keys.firstOrNull() ?: "uri" + val uriData = + AppFunctionData.Builder(objectType, components) + .setString(uriPropertyName, value) + .build() + builder.setAppFunctionData(name, uriData) + } else { + val objData = convertObject(objectType, value as Map, components) + builder.setAppFunctionData(name, objData) + } } } } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/chat/SendMessageUseCase.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/chat/SendMessageUseCase.kt index 91c3518..4f9ac22 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/chat/SendMessageUseCase.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/chat/SendMessageUseCase.kt @@ -16,6 +16,7 @@ package com.example.appfunctions.agent.domain.chat import com.example.appfunctions.agent.data.ChatRepository +import com.example.appfunctions.agent.data.db.entities.MessageAttachment import com.example.appfunctions.agent.data.db.entities.MessageEntity import com.example.appfunctions.agent.data.db.entities.MessageProcessingStatus import com.example.appfunctions.agent.data.db.entities.MessageRole @@ -43,6 +44,7 @@ class SendMessageUseCase processingStatus: MessageProcessingStatus, pendingIntentId: String? = null, targetPackageName: String? = null, + attachments: List = emptyList(), ) { val message = MessageEntity( @@ -54,6 +56,7 @@ class SendMessageUseCase processingStatus = processingStatus, pendingIntentId = pendingIntentId, targetPackageName = targetPackageName, + attachments = attachments, ) chatRepository.sendMessage(message) } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt index 327c8cf..193e37c 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt @@ -88,6 +88,7 @@ import androidx.compose.ui.input.key.KeyEventType import androidx.compose.ui.input.key.key import androidx.compose.ui.input.key.onPreviewKeyEvent import androidx.compose.ui.input.key.type +import androidx.compose.ui.layout.ContentScale import androidx.compose.ui.platform.LocalConfiguration import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.platform.LocalDensity @@ -118,6 +119,7 @@ import androidx.compose.ui.window.PopupProperties import androidx.core.graphics.drawable.toBitmap import androidx.hilt.navigation.compose.hiltViewModel import androidx.lifecycle.compose.collectAsStateWithLifecycle +import coil.compose.AsyncImage import com.example.appfunctions.agent.R import com.example.appfunctions.agent.data.LlmModel import com.example.appfunctions.agent.data.db.entities.MessageEntity @@ -669,15 +671,19 @@ fun MessageBubble( Spacer(modifier = Modifier.width(8.dp)) } val contentText = - if (message.textContent.isEmpty() && + if ( + message.textContent.isEmpty() && message.pendingIntentId != null ) { stringResource(R.string.agent_demo_action_confirmation_needed) } else { message.textContent } + if (message.role != MessageRole.USER) { - Markdown(content = contentText) + if (contentText.isNotEmpty()) { + Markdown(content = contentText) + } } else { val chipBgColor = MaterialTheme.colorScheme.primary val chipTextColor = MaterialTheme.colorScheme.onPrimary @@ -704,7 +710,10 @@ fun MessageBubble( "|", ) { Regex.escape(it.label) } val regex = - Regex("@($appLabelsPattern)\\b", RegexOption.IGNORE_CASE) + Regex( + "@($appLabelsPattern)\\b", + RegexOption.IGNORE_CASE, + ) regex.findAll(contentText).forEachIndexed { index, match -> val id = "chip_$index" val appName = match.value @@ -795,6 +804,24 @@ fun MessageBubble( } } } + + if (message.role != MessageRole.USER) { + message.attachments.forEach { attachment -> + if (attachment.mimeType.startsWith("image/", ignoreCase = true)) { + Spacer(modifier = Modifier.height(6.dp)) + AsyncImage( + model = attachment.uri, + contentDescription = "Generated Image", + modifier = + Modifier + .fillMaxWidth(0.85f) + .height(240.dp) + .clip(RoundedCornerShape(16.dp)), + contentScale = ContentScale.Crop, + ) + } + } + } } } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/debugging/FunctionsFoundContent.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/debugging/FunctionsFoundContent.kt index 146d1f6..ca95ccb 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/debugging/FunctionsFoundContent.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/debugging/FunctionsFoundContent.kt @@ -65,7 +65,6 @@ import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.text.withStyle import androidx.compose.ui.tooling.preview.Preview import androidx.compose.ui.unit.dp -import androidx.appfunctions.AppFunctionException import com.example.appfunctions.agent.R import com.example.appfunctions.agent.domain.appfunction.AppFunctionExceptionFormatter import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionResult diff --git a/agent/app/src/main/res/xml/file_paths.xml b/agent/app/src/main/res/xml/file_paths.xml new file mode 100644 index 0000000..5074b23 --- /dev/null +++ b/agent/app/src/main/res/xml/file_paths.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt index 2816960..62cb16a 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt @@ -408,4 +408,116 @@ class GeminiToolConverterTest { assertEquals(expectedJson, schema) } + + @Test + fun convert_stringParameterEndingWithUri_injectsFileReferenceFormat() { + val parameter = + AppFunctionParameterMetadata( + name = "wallpaperUri", + isRequired = true, + dataType = AppFunctionStringTypeMetadata(isNullable = false), + description = "A URI parameter", + ) + val tool = + AppFunctionMetadata( + id = "com.example.my_function", + packageName = "com.example", + isEnabled = true, + schema = null, + parameters = listOf(parameter), + response = + AppFunctionResponseMetadata( + valueType = AppFunctionStringTypeMetadata(isNullable = false), + description = "", + ), + components = AppFunctionComponentsMetadata(emptyMap()), + description = "Test function", + deprecation = null, + ) + + val schema = converter.convert(tool) + + val expectedJson = + Json.parseToJsonElement( + """ + { + "name": "com_example_my_function", + "description": "Test function", + "parameters": { + "type": "object", + "properties": { + "wallpaperUri": { + "type": "string", + "format": "file_reference", + "description": "A URI parameter" + } + }, + "required": ["wallpaperUri"] + } + } + """, + ) + + assertEquals(expectedJson, schema) + } + + @Test + fun convert_uriObjectType_injectsFileReferenceFormat() { + val uriObjectMetadata = + AppFunctionObjectTypeMetadata( + properties = emptyMap(), + required = emptyList(), + qualifiedName = "android.net.Uri", + isNullable = false, + description = "Uri object", + ) + val parameter = + AppFunctionParameterMetadata( + name = "customUri", + isRequired = true, + dataType = uriObjectMetadata, + description = "A Uri object parameter", + ) + val tool = + AppFunctionMetadata( + id = "com.example.my_function", + packageName = "com.example", + isEnabled = true, + schema = null, + parameters = listOf(parameter), + response = + AppFunctionResponseMetadata( + valueType = AppFunctionStringTypeMetadata(isNullable = false), + description = "", + ), + components = AppFunctionComponentsMetadata(emptyMap()), + description = "Test function", + deprecation = null, + ) + + val schema = converter.convert(tool) + + val expectedJson = + Json.parseToJsonElement( + """ + { + "name": "com_example_my_function", + "description": "Test function", + "parameters": { + "type": "object", + "properties": { + "customUri": { + "type": "string", + "format": "file_reference", + "description": "A Uri object parameter" + } + }, + "required": ["customUri"] + } + } + """, + ) + + assertEquals(expectedJson, schema) + } } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/db/entities/MessageAttachmentConverterTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/db/entities/MessageAttachmentConverterTest.kt new file mode 100644 index 0000000..7a7994e --- /dev/null +++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/db/entities/MessageAttachmentConverterTest.kt @@ -0,0 +1,55 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.data.db.entities + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class MessageAttachmentConverterTest { + private val converter = MessageAttachmentConverter() + + @Test + fun testSerializationAndDeserialization() { + val attachments = + listOf( + MessageAttachment( + uri = "content://com.example.appfunctions.agent.fileprovider/cache/test.jpg", + mimeType = "image/jpeg", + ), + MessageAttachment( + uri = "content://com.example.appfunctions.agent.fileprovider/cache/test.png", + mimeType = "image/png", + ), + ) + + val json = converter.fromAttachments(attachments) + val decoded = converter.toAttachments(json) + + assertEquals(attachments, decoded) + } + + @Test + fun testEmptyOrNullStringDeserializesToEmptyList() { + assertTrue(converter.toAttachments(null).isEmpty()) + assertTrue(converter.toAttachments("").isEmpty()) + } + + @Test + fun testMalformedJsonDeserializesToEmptyList() { + assertTrue(converter.toAttachments("{invalid_json").isEmpty()) + } +} diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt index 4dbf97d..e8f78da 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt @@ -15,11 +15,14 @@ */ package com.example.appfunctions.agent.domain +import android.content.Intent +import androidx.appfunctions.AppFunctionData import androidx.appfunctions.metadata.AppFunctionMetadata import androidx.appfunctions.metadata.AppFunctionPackageMetadata import com.example.appfunctions.agent.data.LlmModel import com.example.appfunctions.agent.data.LlmProviderName import com.example.appfunctions.agent.data.SettingsRepository +import com.example.appfunctions.agent.data.db.entities.MessageAttachment import com.example.appfunctions.agent.data.db.entities.MessageEntity import com.example.appfunctions.agent.data.db.entities.MessageProcessingStatus import com.example.appfunctions.agent.data.db.entities.MessageRole @@ -64,6 +67,7 @@ class AgentOrchestratorTest { private val executeAppFunctionUseCase: ExecuteAppFunctionUseCase = mockk() private val sendMessageUseCase: SendMessageUseCase = mockk(relaxed = true) private val convertInputToAppFunctionDataUseCase: ConvertInputToAppFunctionDataUseCase = mockk() + private val context: android.content.Context = mockk(relaxed = true) private val savePendingIntentUseCase: SavePendingIntentUseCase = mockk(relaxed = true) private lateinit var agentOrchestrator: AgentOrchestrator @@ -72,6 +76,7 @@ class AgentOrchestratorTest { fun setUp() { agentOrchestrator = AgentOrchestrator( + context = context, manageThreadsUseCase = manageThreadsUseCase, observePendingMessagesUseCase = observePendingMessagesUseCase, sendMessageUseCase = sendMessageUseCase, @@ -203,7 +208,8 @@ class AgentOrchestratorTest { val llmProvider = mockk() val tool1 = createMockTool("com.google.android.appfunctiontestingagent", "run_geo_code") - val tool2 = createMockTool("com.google.android.digitalwellbeing", "digital_well_being_tool") + val tool2 = + createMockTool("com.google.android.digitalwellbeing", "digital_well_being_tool") mockAppFunctions(listOf(tool1, tool2)) setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) @@ -234,7 +240,8 @@ class AgentOrchestratorTest { val llmProvider = mockk() val tool1 = createMockTool("com.google.android.appfunctiontestingagent", "run_geo_code") - val tool2 = createMockTool("com.google.android.digitalwellbeing", "digital_well_being_tool") + val tool2 = + createMockTool("com.google.android.digitalwellbeing", "digital_well_being_tool") mockAppFunctions(listOf(tool1, tool2)) setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) @@ -282,17 +289,18 @@ class AgentOrchestratorTest { coEvery { llmProvider.generateResponse(null, any(), any(), any(), any()) - } returns LlmResponse.Success( - "interaction_1", - listOf( - LlmResponsePart.ToolCall( - packageName = "com.example.calendar", - functionId = "create_event", - arguments = emptyMap(), - callId = "call_1", - ) + } returns + LlmResponse.Success( + "interaction_1", + listOf( + LlmResponsePart.ToolCall( + packageName = "com.example.calendar", + functionId = "create_event", + arguments = emptyMap(), + callId = "call_1", + ), + ), ) - ) val expectedErrorOutput = "Tool execution failed for create_event: Error: AppFunctionPermissionRequiredException - Calendar permission required" @@ -305,17 +313,18 @@ class AgentOrchestratorTest { coVerify { llmProvider.generateResponse( previousInteractionId = eq("interaction_1"), - input = eq( - LlmInput.ToolResponse( - listOf( - ToolOutput( - functionId = "create_event", - callId = "call_1", - result = expectedErrorOutput, - ) - ) - ) - ), + input = + eq( + LlmInput.ToolResponse( + listOf( + ToolOutput( + functionId = "create_event", + callId = "call_1", + result = expectedErrorOutput, + ), + ), + ), + ), tools = listOf(tool1), apiKey = "dummy_key", modelName = any(), @@ -354,7 +363,7 @@ class AgentOrchestratorTest { id: String, isEnabled: Boolean = true, ): AppFunctionMetadata { - val tool = mockk() + val tool = mockk(relaxed = true) every { tool.packageName } returns packageName every { tool.id } returns id every { tool.isEnabled } returns isEnabled @@ -384,4 +393,164 @@ class AgentOrchestratorTest { coEvery { settingsRepository.disconnectedApps } returns flowOf(disconnectedApps) coEvery { llmProviderFactory.getProvider(LlmProviderName.GEMINI) } returns llmProvider } + + @Test + fun `observeAndProcessMessages extracts attachments when tool returns imageUri and mimeType`() = + runTest { + val threadId = "thread_1" + val message = createUserMessage(threadId, "generate image of a dog") + val thread = createThread(threadId) + val llmProvider = mockk() + + val generateTool = createMockTool("com.example.appfunctions.agent", "generateImage") + mockAppFunctions(listOf(generateTool)) + setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) + + val toolCall = + LlmResponsePart.ToolCall( + packageName = "com.example.appfunctions.agent", + functionId = "generateImage", + arguments = mapOf("prompt" to "dog"), + callId = "call_1", + ) + + val firstResponse = + LlmResponse.Success( + interactionId = "interaction_1", + parts = listOf(toolCall), + ) + val secondResponse = + LlmResponse.Success( + interactionId = "interaction_2", + parts = listOf(LlmResponsePart.Text("Here is your image!")), + ) + + coEvery { + llmProvider.generateResponse( + previousInteractionId = null, + input = any(), + tools = any(), + apiKey = any(), + modelName = any(), + ) + } returns firstResponse + + coEvery { + llmProvider.generateResponse( + previousInteractionId = "interaction_1", + input = any(), + tools = any(), + apiKey = any(), + modelName = any(), + ) + } returns secondResponse + + coEvery { + convertInputToAppFunctionDataUseCase(any(), any(), any()) + } returns Result.success(AppFunctionData.EMPTY) + + coEvery { + executeAppFunctionUseCase(any(), any(), any()) + } returns + ExecuteAppFunctionResult.Data( + data = AppFunctionData.EMPTY, + formattedJson = + """{"imageUri":"content://com.example.appfunctions.agent.fileprovider""" + + """/cache/test.jpg","mimeType":"image/jpeg","prompt":"dog"}""", + ) + + agentOrchestrator.observeAndProcessMessages(threadId) + + coVerify { + sendMessageUseCase( + threadId = threadId, + role = MessageRole.ASSISTANT, + textContent = "Here is your image!", + processingStatus = MessageProcessingStatus.PROCESSED, + pendingIntentId = null, + targetPackageName = null, + attachments = + listOf( + MessageAttachment( + uri = + "content://com.example.appfunctions.agent.fileprovider" + + "/cache/test.jpg", + mimeType = "image/jpeg", + ), + ), + ) + } + } + + @Test + fun `observeAndProcessMessages grants URI read permission when tool is called with content URI argument`() = + runTest { + val threadId = "thread_1" + val message = createUserMessage(threadId, "set wallpaper") + val thread = createThread(threadId) + val llmProvider = mockk() + + val targetTool = createMockTool("com.example.targetapp", "setWallpaper") + mockAppFunctions(listOf(targetTool)) + setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) + + val contentUri = "content://com.example.appfunctions.agent.fileprovider/cache/img.jpg" + val toolCall = + LlmResponsePart.ToolCall( + packageName = "com.example.targetapp", + functionId = "setWallpaper", + arguments = mapOf("uri" to contentUri), + callId = "call_2", + ) + + coEvery { + llmProvider.generateResponse( + previousInteractionId = null, + input = any(), + tools = any(), + apiKey = any(), + modelName = any(), + ) + } returns + LlmResponse.Success( + interactionId = "interaction_1", + parts = listOf(toolCall), + ) + + coEvery { + llmProvider.generateResponse( + previousInteractionId = "interaction_1", + input = any(), + tools = any(), + apiKey = any(), + modelName = any(), + ) + } returns + LlmResponse.Success( + interactionId = "interaction_2", + parts = listOf(LlmResponsePart.Text("Wallpaper set")), + ) + + coEvery { + convertInputToAppFunctionDataUseCase(any(), any(), any()) + } returns Result.success(AppFunctionData.EMPTY) + + coEvery { + executeAppFunctionUseCase(any(), any(), any()) + } returns + ExecuteAppFunctionResult.Data( + data = AppFunctionData.EMPTY, + formattedJson = """{"success":true}""", + ) + + agentOrchestrator.observeAndProcessMessages(threadId) + + coVerify { + context.grantUriPermission( + eq("com.example.targetapp"), + any(), + eq(Intent.FLAG_GRANT_READ_URI_PERMISSION), + ) + } + } } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/AppFunctionExceptionFormatterTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/AppFunctionExceptionFormatterTest.kt index b60f919..906a3dd 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/AppFunctionExceptionFormatterTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/AppFunctionExceptionFormatterTest.kt @@ -86,7 +86,8 @@ class AppFunctionExceptionFormatterTest { "com.example.chatapp.appfunctions.BaseChatAppFunctionService#send", ) assertEquals( - "Tool execution failed for com.example.chatapp.appfunctions.BaseChatAppFunctionService#send: Error: AppFunctionInvalidArgumentException - Message body cannot be empty", + "Tool execution failed for com.example.chatapp.appfunctions.BaseChatAppFunctionService#send: " + + "Error: AppFunctionInvalidArgumentException - Message body cannot be empty", formatted, ) } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCaseTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCaseTest.kt index 25760fb..478030b 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCaseTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCaseTest.kt @@ -238,4 +238,32 @@ class ConvertInputToAppFunctionDataUseCaseTest { assertEquals(true, result.isFailure) } + + @Test + fun convert_uriObjectTypeWithStringInput_buildsUriAppFunctionData() { + val uriObjectType = + AppFunctionObjectTypeMetadata( + properties = + mapOf( + "uri" to AppFunctionStringTypeMetadata(false), + ), + required = listOf("uri"), + qualifiedName = "android.net.Uri", + isNullable = false, + ) + val parameters = + listOf( + AppFunctionParameterMetadata( + name = "wallpaperUri", + isRequired = true, + dataType = uriObjectType, + ), + ) + val inputs = mapOf("wallpaperUri" to "content://com.example/file.jpg") + + val result = useCase(parameters, components, inputs).getOrThrow() + + val uriData = result.getAppFunctionData("wallpaperUri") + assertEquals("content://com.example/file.jpg", uriData?.getString("uri")) + } } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ExecuteAppFunctionUseCaseTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ExecuteAppFunctionUseCaseTest.kt index 7d4db46..9c660ff 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ExecuteAppFunctionUseCaseTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ExecuteAppFunctionUseCaseTest.kt @@ -38,38 +38,39 @@ class ExecuteAppFunctionUseCaseTest { private val convertAppFunctionDataToJsonUseCase = mockk() private val useCase = ExecuteAppFunctionUseCase(appFunctionManager, convertAppFunctionDataToJsonUseCase) - @Test - fun `invoke returns Error with original AppFunctionException when response is Error`() = runTest { - val function = mockk(relaxed = true) - every { function.packageName } returns "com.example.app" - every { function.id } returns "test_function" - val parameters = mockk() + fun `invoke returns Error with original AppFunctionException when response is Error`() = + runTest { + val function = mockk(relaxed = true) + every { function.packageName } returns "com.example.app" + every { function.id } returns "test_function" + val parameters = mockk() - val appException = AppFunctionPermissionRequiredException("Permission needed") - coEvery { appFunctionManager.executeAppFunction(any()) } returns ExecuteAppFunctionResponse.Error(appException) + val appException = AppFunctionPermissionRequiredException("Permission needed") + coEvery { appFunctionManager.executeAppFunction(any()) } returns ExecuteAppFunctionResponse.Error(appException) - val result = useCase(function, parameters) + val result = useCase(function, parameters) - assertTrue(result is ExecuteAppFunctionResult.Error) - val errorResult = result as ExecuteAppFunctionResult.Error - assertEquals(appException, errorResult.exception) - } + assertTrue(result is ExecuteAppFunctionResult.Error) + val errorResult = result as ExecuteAppFunctionResult.Error + assertEquals(appException, errorResult.exception) + } @Test - fun `invoke returns Error when executeAppFunction throws exception`() = runTest { - val function = mockk(relaxed = true) - every { function.packageName } returns "com.example.app" - every { function.id } returns "test_function" - val parameters = mockk() + fun `invoke returns Error when executeAppFunction throws exception`() = + runTest { + val function = mockk(relaxed = true) + every { function.packageName } returns "com.example.app" + every { function.id } returns "test_function" + val parameters = mockk() - val runtimeException = RuntimeException("Unexpected crash") - coEvery { appFunctionManager.executeAppFunction(any()) } throws runtimeException + val runtimeException = RuntimeException("Unexpected crash") + coEvery { appFunctionManager.executeAppFunction(any()) } throws runtimeException - val result = useCase(function, parameters) + val result = useCase(function, parameters) - assertTrue(result is ExecuteAppFunctionResult.Error) - val errorResult = result as ExecuteAppFunctionResult.Error - assertEquals(runtimeException, errorResult.exception) - } + assertTrue(result is ExecuteAppFunctionResult.Error) + val errorResult = result as ExecuteAppFunctionResult.Error + assertEquals(runtimeException, errorResult.exception) + } } diff --git a/agent/gradle/libs.versions.toml b/agent/gradle/libs.versions.toml index a485239..907b886 100644 --- a/agent/gradle/libs.versions.toml +++ b/agent/gradle/libs.versions.toml @@ -91,6 +91,7 @@ spotless = { id = "com.diffplug.spotless", version.ref = "spotless" } ksp = { id = "com.google.devtools.ksp", version.ref = "ksp" } screenshot = { id = "com.android.compose.screenshot", version.ref = "screenshot" } hilt = { id = "com.google.dagger.hilt.android", version.ref = "hilt" } +kotlin-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlin" } oss-licenses = { id = "com.google.android.gms.oss-licenses-plugin", version.ref = "ossLicensesPlugin" }