From f8b320046448318ba6f1ede51ef5085d01eaf569 Mon Sep 17 00:00:00 2001 From: ZXCLI Date: Fri, 11 Sep 2026 00:31:00 +0800 Subject: [PATCH] android: visible-light PIP overlay (camera2, draggable, 3 sizes) --- android/app/src/main/AndroidManifest.xml | 3 + .../com/mag160c/thermal/ui/live/LiveScreen.kt | 157 +++++++++++++- .../mag160c/thermal/ui/live/LiveViewModel.kt | 35 ++++ .../mag160c/thermal/ui/live/PipCameraView.kt | 195 ++++++++++++++++++ android/app/src/main/res/drawable/ic_pip.xml | 4 + build-artifacts/mag160c-app-debug.apk | 4 +- docs/android_app/real_device_checklist.md | 12 ++ docs/android_app/session_state.md | 12 +- 8 files changed, 418 insertions(+), 4 deletions(-) create mode 100644 android/app/src/main/kotlin/com/mag160c/thermal/ui/live/PipCameraView.kt create mode 100644 android/app/src/main/res/drawable/ic_pip.xml diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index 710d666..167afab 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -2,6 +2,9 @@ + + + diff --git a/android/app/src/main/kotlin/com/mag160c/thermal/ui/live/LiveScreen.kt b/android/app/src/main/kotlin/com/mag160c/thermal/ui/live/LiveScreen.kt index 0a1280f..5f78d4c 100644 --- a/android/app/src/main/kotlin/com/mag160c/thermal/ui/live/LiveScreen.kt +++ b/android/app/src/main/kotlin/com/mag160c/thermal/ui/live/LiveScreen.kt @@ -1,9 +1,13 @@ package com.mag160c.thermal.ui.live import android.view.SurfaceView +import android.view.TextureView +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.result.contract.ActivityResultContracts import androidx.compose.foundation.background import androidx.compose.foundation.border import androidx.compose.foundation.clickable +import androidx.compose.foundation.gestures.detectDragGestures import androidx.compose.foundation.gestures.detectTapGestures import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box @@ -31,6 +35,7 @@ import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.runtime.Composable +import androidx.compose.runtime.DisposableEffect import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.collectAsState import androidx.compose.runtime.getValue @@ -39,6 +44,7 @@ import androidx.compose.runtime.remember import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip import androidx.compose.ui.graphics.Color import androidx.compose.ui.graphics.graphicsLayer import androidx.compose.ui.input.pointer.pointerInput @@ -48,6 +54,7 @@ import androidx.compose.ui.platform.LocalDensity import androidx.compose.ui.res.painterResource import androidx.compose.ui.unit.dp import androidx.compose.ui.viewinterop.AndroidView +import androidx.core.content.ContextCompat import androidx.lifecycle.viewmodel.compose.viewModel import com.mag160c.thermal.R import com.mag160c.thermal.core.Palettes @@ -72,6 +79,18 @@ fun LiveScreen(vm: LiveViewModel = viewModel(), onOpenGallery: () -> Unit = {}) var showPalette by remember { mutableStateOf(false) } var navPx by remember { mutableStateOf(UiInsets.navPx) } var shutterPx by remember { mutableStateOf(0) } + // visible-light PIP (Phase E) + var pipEngine by remember { mutableStateOf(null) } + var viewportSize by remember { mutableStateOf(androidx.compose.ui.unit.IntSize.Zero) } + val cameraPermission = rememberLauncherForActivityResult( + ActivityResultContracts.RequestPermission(), + ) { granted -> + if (granted) { + vm.togglePip() + } else { + com.mag160c.thermal.media.DebugLog.log("pip", "camera permission denied by user") + } + } LaunchedEffect(Unit) { vm.connect() @@ -84,7 +103,26 @@ fun LiveScreen(vm: LiveViewModel = viewModel(), onOpenGallery: () -> Unit = {}) } } - Box(modifier = Modifier.fillMaxSize()) { + // The PIP overlay owns the camera: it releases on any of these exits — + // PIP switched off (overlay leaves composition), live screen left, or the + // activity going to the background (ON_STOP). + val lifecycleOwner = androidx.lifecycle.compose.LocalLifecycleOwner.current + DisposableEffect(lifecycleOwner) { + val observer = androidx.lifecycle.LifecycleEventObserver { _, event -> + if (event == androidx.lifecycle.Lifecycle.Event.ON_STOP) { + pipEngine?.release() + pipEngine = null + } + } + lifecycleOwner.lifecycle.addObserver(observer) + onDispose { lifecycleOwner.lifecycle.removeObserver(observer) } + } + + Box( + modifier = Modifier + .fillMaxSize() + .onSizeChanged { viewportSize = it }, + ) { AndroidSurface(vm) Box( modifier = Modifier @@ -194,9 +232,54 @@ fun LiveScreen(vm: LiveViewModel = viewModel(), onOpenGallery: () -> Unit = {}) ) } } + Box( + modifier = Modifier.weight(1f).graphicsLayer { rotationZ = -phi.toFloat() }, + contentAlignment = Alignment.Center, + ) { + Column( + horizontalAlignment = Alignment.CenterHorizontally, + modifier = Modifier + .clickable { + if (state.pipOn) { + vm.togglePip() + } else if (ContextCompat.checkSelfPermission( + context, android.Manifest.permission.CAMERA, + ) == android.content.pm.PackageManager.PERMISSION_GRANTED + ) { + vm.togglePip() + } else { + cameraPermission.launch(android.Manifest.permission.CAMERA) + } + } + .padding(4.dp), + ) { + val pipOn = state.pipOn + Icon( + painterResource(R.drawable.ic_pip), "画中画", + tint = if (pipOn) MaterialTheme.colorScheme.primary + else MaterialTheme.colorScheme.onSurfaceVariant, + ) + Text( + if (pipOn) "画中画·开" else "画中画", + style = MaterialTheme.typography.labelSmall, + color = if (pipOn) MaterialTheme.colorScheme.primary + else MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + } } } + // ---- visible-light PIP overlay (before the shutter row so the row draws on top) ---- + if (state.pipOn) { + PipOverlay( + state = state, + vm = vm, + viewport = viewportSize, + onEngine = { pipEngine = it }, + ) + } + // ---- camera shutter row above the bottom navigation (live tab) ---- val recording = state.status == "recording" Row( @@ -289,6 +372,78 @@ fun LiveScreen(vm: LiveViewModel = viewModel(), onOpenGallery: () -> Unit = {}) } } +/** + * Draggable visible-light PIP overlay (Phase E). + * + * Position is expressed relative to the thermal viewport (the same rect the + * renderer draws into): left = viewport.left + xf * free width, with the + * color-bar footprint kept clear at the right edge. Drag updates the stored + * fractions; tap cycles 96/128/160 dp; double tap closes. + */ +@Composable +private fun PipOverlay( + state: LiveViewModel.LiveState, + vm: LiveViewModel, + viewport: androidx.compose.ui.unit.IntSize, + onEngine: (PipCameraEngine) -> Unit, +) { + val density = LocalDensity.current + val context = LocalContext.current + val engine = remember { PipCameraEngine(context.applicationContext) } + LaunchedEffect(Unit) { onEngine(engine) } + // switching PIP off / leaving the screen removes this overlay: release here + DisposableEffect(engine) { + onDispose { engine.release() } + } + + val wDp = LiveViewModel.PIP_WIDTHS_DP[state.pipSizeIndex.coerceIn(0, 2)] + val hDp = wDp * 3 / 4 + + with(density) { + val top = vm.uiTopPx.toFloat() + val bottom = (viewport.height - vm.uiBottomPx).toFloat().coerceAtLeast(top + 1f) + val freeW = (viewport.width - wDp * density.density).coerceAtLeast(1f) + val freeH = (bottom - top - hDp * density.density).coerceAtLeast(1f) + // xf = 1 parks the PIP at the right edge minus the color-bar margin + val rightMargin = LiveViewModel.PIP_RIGHT_MARGIN_DP * density.density + val left = vm.state.value.pipXf * (freeW - rightMargin) + val topPx = top + vm.state.value.pipYf * freeH + + Box( + modifier = Modifier + .offset(x = (left / density.density).dp, y = (topPx / density.density).dp) + .size(wDp.dp, hDp.dp) + .border(2.dp, Color.White, RoundedCornerShape(8.dp)) + .clip(RoundedCornerShape(8.dp)) + .pointerInput(state.pipSizeIndex) { + detectDragGestures { change, drag -> + change.consume() + val s = vm.state.value + val nx = (s.pipXf + drag.x / (freeW - rightMargin).coerceAtLeast(1f)).coerceIn(0f, 1f) + val ny = (s.pipYf + drag.y / freeH).coerceIn(0f, 1f) + vm.setPipPos(nx, ny) + } + } + .pointerInput(state.pipSizeIndex) { + detectTapGestures( + onTap = { vm.cyclePipSize() }, + onDoubleTap = { vm.togglePip() }, + ) + }, + ) { + AndroidView( + factory = { ctx -> + TextureView(ctx).also { tv -> + tv.surfaceTextureListener = engine + engine.attach(tv) + } + }, + modifier = Modifier.fillMaxSize(), + ) + } + } +} + private fun statusText(state: LiveViewModel.LiveState): String = when (state.status) { "no_device" -> "未检测到热像仪，请插入MAG160C" "no_permission" -> "USB权限未授予" diff --git a/android/app/src/main/kotlin/com/mag160c/thermal/ui/live/LiveViewModel.kt b/android/app/src/main/kotlin/com/mag160c/thermal/ui/live/LiveViewModel.kt index 46085eb..9e6c4de 100644 --- a/android/app/src/main/kotlin/com/mag160c/thermal/ui/live/LiveViewModel.kt +++ b/android/app/src/main/kotlin/com/mag160c/thermal/ui/live/LiveViewModel.kt @@ -27,6 +27,14 @@ class LiveViewModel(app: Application) : AndroidViewModel(app) { val identity: IrSession.CameraIdentity? = null, val maxTraceOn: Boolean = true, val probes: List = emptyList(), + /** Visible-light PIP overlay (Phase E). */ + val pipOn: Boolean = false, + /** 0/1/2 -> 96/128/160 dp wide. */ + val pipSizeIndex: Int = 1, + /** PIP top-left X as a fraction of the free space (0..1). */ + val pipXf: Float = 1f, + /** PIP top-left Y as a fraction of the free space below the top bar (0..1). */ + val pipYf: Float = 0f, ) private val _state = MutableStateFlow(LiveState()) @@ -39,6 +47,14 @@ class LiveViewModel(app: Application) : AndroidViewModel(app) { private val session = IrSession(app) + companion object { + /** PIP overlay width in dp for size index 0/1/2. */ + val PIP_WIDTHS_DP = intArrayOf(96, 128, 160) + + /** Color-bar footprint kept clear at the right edge (20dp bar + 2x12dp). */ + const val PIP_RIGHT_MARGIN_DP = 32 + } + private var usbReceiver: android.content.BroadcastReceiver? = null private var recorder: com.mag160c.thermal.media.Mp4Recorder? = null @@ -200,6 +216,25 @@ class LiveViewModel(app: Application) : AndroidViewModel(app) { _state.value = _state.value.copy(maxTraceOn = !_state.value.maxTraceOn) } + // ---- visible-light PIP (Phase E) ---- + + /** PIP width options in dp (matches [PIP_WIDTHS_DP]). */ + fun togglePip() { + _state.value = _state.value.copy(pipOn = !_state.value.pipOn) + } + + fun cyclePipSize() { + _state.value = _state.value.copy(pipSizeIndex = (_state.value.pipSizeIndex + 1) % 3) + } + + /** Drag position as fractions of the free space (clamped to 0..1). */ + fun setPipPos(xf: Float, yf: Float) { + _state.value = _state.value.copy( + pipXf = xf.coerceIn(0f, 1f), + pipYf = yf.coerceIn(0f, 1f), + ) + } + /** Capture: rendered JPEG + raw frame + camera info -> MDT -> MediaStore. */ fun capturePhoto(context: android.content.Context) { val frame = latestFrame ?: return diff --git a/android/app/src/main/kotlin/com/mag160c/thermal/ui/live/PipCameraView.kt b/android/app/src/main/kotlin/com/mag160c/thermal/ui/live/PipCameraView.kt new file mode 100644 index 0000000..ec97377 --- /dev/null +++ b/android/app/src/main/kotlin/com/mag160c/thermal/ui/live/PipCameraView.kt @@ -0,0 +1,195 @@ +package com.mag160c.thermal.ui.live + +import android.Manifest +import android.content.Context +import android.content.pm.PackageManager +import android.graphics.SurfaceTexture +import android.hardware.camera2.CameraCaptureSession +import android.hardware.camera2.CameraCharacteristics +import android.hardware.camera2.CameraDevice +import android.hardware.camera2.CameraManager +import android.hardware.camera2.CaptureRequest +import android.os.Handler +import android.os.HandlerThread +import android.view.Surface +import android.view.TextureView +import com.mag160c.thermal.media.DebugLog + +/** + * Minimal Camera2 preview engine for the visible-light PIP overlay (Phase E). + * + * Design rules: + * - open the first BACK-facing camera, preview-only (TEMPLATE_PREVIEW), no + * capture requests, no recording, no image reader; + * - every failure path logs under tag "pip" and tears the camera down — + * a camera problem must never crash or block the thermal live view; + * - [release] is idempotent and safe to call from any state. + */ +class PipCameraEngine(private val context: Context) : TextureView.SurfaceTextureListener { + + private var textureView: TextureView? = null + private var camera: CameraDevice? = null + private var session: CameraCaptureSession? = null + private var thread: HandlerThread? = null + private var handler: Handler? = null + private var surface: Surface? = null + + @Volatile + private var released = false + + /** True once a repeating preview request has been submitted. */ + @Volatile + var previewing = false + private set + + /** Attach to a TextureView and start the preview as soon as it has a surface. */ + fun attach(view: TextureView) { + try { + textureView = view + view.surfaceTextureListener = this + if (view.isAvailable) onSurfaceTextureAvailable(view.surfaceTexture!!, view.width, view.height) + } catch (e: Exception) { + DebugLog.log("pip", "attach failed: $e") + } + } + + override fun onSurfaceTextureAvailable(st: SurfaceTexture, width: Int, height: Int) { + try { + startThread() + surface = Surface(st) + openCamera() + } catch (e: Exception) { + DebugLog.log("pip", "surface available handling failed: $e") + release() + } + } + + override fun onSurfaceTextureSizeChanged(st: SurfaceTexture, width: Int, height: Int) = Unit + + override fun onSurfaceTextureDestroyed(st: SurfaceTexture): Boolean { + release() + return true + } + + override fun onSurfaceTextureUpdated(st: SurfaceTexture) = Unit + + private fun startThread() { + if (thread == null) { + val t = HandlerThread("pip-camera") + t.start() + thread = t + handler = Handler(t.looper) + } + } + + private fun openCamera() { + val manager = context.getSystemService(Context.CAMERA_SERVICE) as CameraManager + val id = try { + manager.cameraIdList.firstOrNull { cid -> + val facing = manager.getCameraCharacteristics(cid) + .get(CameraCharacteristics.LENS_FACING) + facing == CameraCharacteristics.LENS_FACING_BACK + } ?: manager.cameraIdList.firstOrNull() + } catch (e: Exception) { + DebugLog.log("pip", "no camera listed: $e") + null + } + if (id == null) { + DebugLog.log("pip", "no camera available") + return + } + if (context.checkSelfPermission(Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) { + DebugLog.log("pip", "camera permission not granted") + return + } + try { + manager.openCamera(id, object : CameraDevice.StateCallback() { + override fun onOpened(device: CameraDevice) { + camera = device + DebugLog.log("pip", "camera opened id=$id") + createSession(device) + } + + override fun onDisconnected(device: CameraDevice) { + DebugLog.log("pip", "camera disconnected") + release() + } + + override fun onError(device: CameraDevice, error: Int) { + DebugLog.log("pip", "camera error=$error") + release() + } + }, handler) + } catch (e: Exception) { + DebugLog.log("pip", "openCamera failed: $e") + release() + } + } + + private fun createSession(device: CameraDevice) { + val target = surface ?: run { + DebugLog.log("pip", "no surface for session") + return + } + try { + @Suppress("DEPRECATION") + device.createCaptureSession(listOf(target), object : CameraCaptureSession.StateCallback() { + override fun onConfigured(s: CameraCaptureSession) { + session = s + startPreview(device, s, target) + } + + override fun onConfigureFailed(s: CameraCaptureSession) { + DebugLog.log("pip", "capture session configure failed") + release() + } + }, handler) + } catch (e: Exception) { + DebugLog.log("pip", "createCaptureSession failed: $e") + release() + } + } + + private fun startPreview(device: CameraDevice, s: CameraCaptureSession, target: Surface) { + try { + val request = device.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW).apply { + addTarget(target) + set(CaptureRequest.CONTROL_MODE, CaptureRequest.CONTROL_MODE_AUTO) + } + s.setRepeatingRequest(request.build(), null, handler) + previewing = true + DebugLog.log("pip", "repeating preview started") + } catch (e: Exception) { + DebugLog.log("pip", "startPreview failed: $e") + release() + } + } + + /** Idempotent teardown: session -> camera -> surface -> thread. */ + fun release() { + if (released && camera == null && session == null) return + released = true + previewing = false + try { + session?.close() + } catch (e: Exception) { + DebugLog.log("pip", "session close failed: $e") + } + session = null + try { + camera?.close() + } catch (e: Exception) { + DebugLog.log("pip", "camera close failed: $e") + } + camera = null + try { + surface?.release() + } catch (_: Exception) { + } + surface = null + thread?.quitSafely() + thread = null + handler = null + DebugLog.log("pip", "released") + } +} diff --git a/android/app/src/main/res/drawable/ic_pip.xml b/android/app/src/main/res/drawable/ic_pip.xml new file mode 100644 index 0000000..75619b2 --- /dev/null +++ b/android/app/src/main/res/drawable/ic_pip.xml @@ -0,0 +1,4 @@ + + + + diff --git a/build-artifacts/mag160c-app-debug.apk b/build-artifacts/mag160c-app-debug.apk index 7500412..76c54c8 100644 --- a/build-artifacts/mag160c-app-debug.apk +++ b/build-artifacts/mag160c-app-debug.apk @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e31763604680b4e45fbb444dbf96e6eac222f2e7b114f4a21e3c41267c6f4218 -size 12236705 +oid sha256:82ed993ac92ba0b00542505569470020b85f75a90b561c5a5ef3d58a63d2eb46 +size 12662902 diff --git a/docs/android_app/real_device_checklist.md b/docs/android_app/real_device_checklist.md index 43f8cfb..ebc1e32 100644 --- a/docs/android_app/real_device_checklist.md +++ b/docs/android_app/real_device_checklist.md @@ -24,6 +24,18 @@ | 10 | 切到"相册"页 | （无 DebugLog；纯 UI） | 列表出现步骤 8 的照片缩略图（MDT 尾部校验通过才显示） | | 11 | 点该照片进入分析页 | （无 DebugLog；纯 UI） | 图片可缩放/平移；调色板重渲染可用；**图像顶部出现温度条**（`中心 x.x℃ 最低 x.x℃ 最高 x.x℃`）；点图任意位置出现白色圆点+温度标签，再点同一点可清除；备注可编辑保存 | | 12 | 分析页生成 PDF 报告 | （无 DebugLog；纯 UI） | 报告文件在 DCIM/MAG160C 或 Download/MAG160C 生成，可打开 | +| 13 | 实时页顶栏第 5 项点"画中画"（首次会弹相机权限，点允许） | `[pip] camera opened id=` → `[pip] repeating preview started` | 右上角（色标条左侧）出现 128×96dp 白框小窗，显示可见光画面；顶栏该项高亮为"画中画·开" | +| 14 | 拖动 PIP 小窗到其它角落；单击小窗（循环 96/128/160dp）；双击小窗关闭 | `[pip] released`（双击关闭时） | 小窗跟手移动且不出界；单击在三档宽度间循环；双击后小窗消失，顶栏恢复"画中画"未开启态；日志有 `released` | +| 15 | PIP 开启时按 Home 回到桌面再返回 | `[pip] released`（进入后台时） | 返回后若 PIP 仍为开启态，画面重新出现（重新 open）；无崩溃 | + +## 相机（PIP）失败时的表现（设计如此，不算 bug） + +PIP 的相机是可选功能，任何相机异常都只记日志并关闭小窗，**不影响热像主画面**： + +- 无相机硬件 / 权限被拒 → `[pip] no camera available` / `[pip] camera permission not granted`， + 小窗空白或未出现； +- 相机被其它应用占用 → `[pip] camera error=` + `[pip] released`； +- 上述情况下主画面、拍照、录像、分析全部照常工作。 ## 失败时的快速定位（沿用第 11 轮起的诊断路径） diff --git a/docs/android_app/session_state.md b/docs/android_app/session_state.md index 2ab9363..6843737 100644 --- a/docs/android_app/session_state.md +++ b/docs/android_app/session_state.md @@ -413,7 +413,17 @@ proguard-rules.pro 新建（-keep cloud.**；原文件缺失但被 build.gradle 引用）。新增 CloudClientTest（4 项，锁"默认关闭/api() 拒绝"契约）。 debug + release(R8) 双构建通过，29 单测全绿。APK 已更新。 -- [ ] Phase E：可见光 PIP 融合 +- [x] Phase E（2026-09-10）：可见光 PIP 融合：Manifest 加 CAMERA 权限 + + camera.any uses-feature（orientation 未动）；LiveState 加 + pipOn/pipSizeIndex/pipXf/pipYf + togglePip/cyclePipSize/setPipPos； + 新增 ui/live/PipCameraView.kt（Camera2 最小实现，TextureView 预览， + 全部异常只记 DebugLog("pip",…) 并 release，绝不崩溃）；顶栏第 5 项 + "画中画"+ ic_pip.xml（双弧圆+右下实心矩形）；浮层三档 96/128/160dp + （高=宽×3/4），位置相对热像视口、右边缘额外留 32dp 色标条位， + 拖动/单击换档/双击关闭；PIP 关闭、离页、ON_STOP 三处释放相机； + 运行时权限用 rememberLauncherForActivityResult。真机项已写入 + real_device_checklist.md（第 13-15 步 + 失败表现）。APK 已更新。 +- [ ] Phase F：网络互连远程预览 ## 里程碑日志