android: visible-light PIP overlay (camera2, draggable, 3 sizes)

2026-09-11 00:31:00 +08:00
parent c34940e6fd
commit f8b3200464
8 changed files with 418 additions and 4 deletions
@@ -2,6 +2,9 @@
 <manifest xmlns:android="http://schemas.android.com/apk/res/android">

    <uses-feature android:name="android.hardware.usb.host" android:required="true" />
+    <!-- Visible-light PIP overlay (Phase E): optional, requested at runtime -->
+    <uses-feature android:name="android.hardware.camera.any" android:required="false" />
+    <uses-permission android:name="android.permission.CAMERA" />
    <uses-permission android:name="android.permission.READ_MEDIA_IMAGES" />
    <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" android:maxSdkVersion="32" />

@@ -1,9 +1,13 @@
 package com.mag160c.thermal.ui.live

 import android.view.SurfaceView
+import android.view.TextureView
+import androidx.activity.compose.rememberLauncherForActivityResult
+import androidx.activity.result.contract.ActivityResultContracts
 import androidx.compose.foundation.background
 import androidx.compose.foundation.border
 import androidx.compose.foundation.clickable
+import androidx.compose.foundation.gestures.detectDragGestures
 import androidx.compose.foundation.gestures.detectTapGestures
 import androidx.compose.foundation.layout.Arrangement
 import androidx.compose.foundation.layout.Box
@@ -31,6 +35,7 @@ import androidx.compose.material3.MaterialTheme
 import androidx.compose.material3.Surface
 import androidx.compose.material3.Text
 import androidx.compose.runtime.Composable
+import androidx.compose.runtime.DisposableEffect
 import androidx.compose.runtime.LaunchedEffect
 import androidx.compose.runtime.collectAsState
 import androidx.compose.runtime.getValue
@@ -39,6 +44,7 @@ import androidx.compose.runtime.remember
 import androidx.compose.runtime.setValue
 import androidx.compose.ui.Alignment
 import androidx.compose.ui.Modifier
+import androidx.compose.ui.draw.clip
 import androidx.compose.ui.graphics.Color
 import androidx.compose.ui.graphics.graphicsLayer
 import androidx.compose.ui.input.pointer.pointerInput
@@ -48,6 +54,7 @@ import androidx.compose.ui.platform.LocalDensity
 import androidx.compose.ui.res.painterResource
 import androidx.compose.ui.unit.dp
 import androidx.compose.ui.viewinterop.AndroidView
+import androidx.core.content.ContextCompat
 import androidx.lifecycle.viewmodel.compose.viewModel
 import com.mag160c.thermal.R
 import com.mag160c.thermal.core.Palettes
@@ -72,6 +79,18 @@ fun LiveScreen(vm: LiveViewModel = viewModel(), onOpenGallery: () -> Unit = {})
    var showPalette by remember { mutableStateOf(false) }
    var navPx by remember { mutableStateOf(UiInsets.navPx) }
    var shutterPx by remember { mutableStateOf(0) }
+    // visible-light PIP (Phase E)
+    var pipEngine by remember { mutableStateOf<PipCameraEngine?>(null) }
+    var viewportSize by remember { mutableStateOf(androidx.compose.ui.unit.IntSize.Zero) }
+    val cameraPermission = rememberLauncherForActivityResult(
+        ActivityResultContracts.RequestPermission(),
+    ) { granted ->
+        if (granted) {
+            vm.togglePip()
+        } else {
+            com.mag160c.thermal.media.DebugLog.log("pip", "camera permission denied by user")
+        }
+    }

    LaunchedEffect(Unit) {
        vm.connect()
@@ -84,7 +103,26 @@ fun LiveScreen(vm: LiveViewModel = viewModel(), onOpenGallery: () -> Unit = {})
        }
    }

-    Box(modifier = Modifier.fillMaxSize()) {
+    // The PIP overlay owns the camera: it releases on any of these exits —
+    // PIP switched off (overlay leaves composition), live screen left, or the
+    // activity going to the background (ON_STOP).
+    val lifecycleOwner = androidx.lifecycle.compose.LocalLifecycleOwner.current
+    DisposableEffect(lifecycleOwner) {
+        val observer = androidx.lifecycle.LifecycleEventObserver { _, event ->
+            if (event == androidx.lifecycle.Lifecycle.Event.ON_STOP) {
+                pipEngine?.release()
+                pipEngine = null
+            }
+        }
+        lifecycleOwner.lifecycle.addObserver(observer)
+        onDispose { lifecycleOwner.lifecycle.removeObserver(observer) }
+    }
+
+    Box(
+        modifier = Modifier
+            .fillMaxSize()
+            .onSizeChanged { viewportSize = it },
+    ) {
        AndroidSurface(vm)
        Box(
            modifier = Modifier
@@ -194,8 +232,53 @@ fun LiveScreen(vm: LiveViewModel = viewModel(), onOpenGallery: () -> Unit = {})
                        )
                    }
                }
+                Box(
+                    modifier = Modifier.weight(1f).graphicsLayer { rotationZ = -phi.toFloat() },
+                    contentAlignment = Alignment.Center,
+                ) {
+                    Column(
+                        horizontalAlignment = Alignment.CenterHorizontally,
+                        modifier = Modifier
+                            .clickable {
+                                if (state.pipOn) {
+                                    vm.togglePip()
+                                } else if (ContextCompat.checkSelfPermission(
+                                        context, android.Manifest.permission.CAMERA,
+                                    ) == android.content.pm.PackageManager.PERMISSION_GRANTED
+                                ) {
+                                    vm.togglePip()
+                                } else {
+                                    cameraPermission.launch(android.Manifest.permission.CAMERA)
                                }
                            }
+                            .padding(4.dp),
+                    ) {
+                        val pipOn = state.pipOn
+                        Icon(
+                            painterResource(R.drawable.ic_pip), "画中画",
+                            tint = if (pipOn) MaterialTheme.colorScheme.primary
+                            else MaterialTheme.colorScheme.onSurfaceVariant,
+                        )
+                        Text(
+                            if (pipOn) "画中画·开" else "画中画",
+                            style = MaterialTheme.typography.labelSmall,
+                            color = if (pipOn) MaterialTheme.colorScheme.primary
+                            else MaterialTheme.colorScheme.onSurfaceVariant,
+                        )
+                    }
+                }
+            }
+        }
+
+        // ---- visible-light PIP overlay (before the shutter row so the row draws on top) ----
+        if (state.pipOn) {
+            PipOverlay(
+                state = state,
+                vm = vm,
+                viewport = viewportSize,
+                onEngine = { pipEngine = it },
+            )
+        }

        // ---- camera shutter row above the bottom navigation (live tab) ----
        val recording = state.status == "recording"
@@ -289,6 +372,78 @@ fun LiveScreen(vm: LiveViewModel = viewModel(), onOpenGallery: () -> Unit = {})
    }
 }

+/**
+ * Draggable visible-light PIP overlay (Phase E).
+ *
+ * Position is expressed relative to the thermal viewport (the same rect the
+ * renderer draws into): left = viewport.left + xf * free width, with the
+ * color-bar footprint kept clear at the right edge.  Drag updates the stored
+ * fractions; tap cycles 96/128/160 dp; double tap closes.
+ */
+@Composable
+private fun PipOverlay(
+    state: LiveViewModel.LiveState,
+    vm: LiveViewModel,
+    viewport: androidx.compose.ui.unit.IntSize,
+    onEngine: (PipCameraEngine) -> Unit,
+) {
+    val density = LocalDensity.current
+    val context = LocalContext.current
+    val engine = remember { PipCameraEngine(context.applicationContext) }
+    LaunchedEffect(Unit) { onEngine(engine) }
+    // switching PIP off / leaving the screen removes this overlay: release here
+    DisposableEffect(engine) {
+        onDispose { engine.release() }
+    }
+
+    val wDp = LiveViewModel.PIP_WIDTHS_DP[state.pipSizeIndex.coerceIn(0, 2)]
+    val hDp = wDp * 3 / 4
+
+    with(density) {
+        val top = vm.uiTopPx.toFloat()
+        val bottom = (viewport.height - vm.uiBottomPx).toFloat().coerceAtLeast(top + 1f)
+        val freeW = (viewport.width - wDp * density.density).coerceAtLeast(1f)
+        val freeH = (bottom - top - hDp * density.density).coerceAtLeast(1f)
+        // xf = 1 parks the PIP at the right edge minus the color-bar margin
+        val rightMargin = LiveViewModel.PIP_RIGHT_MARGIN_DP * density.density
+        val left = vm.state.value.pipXf * (freeW - rightMargin)
+        val topPx = top + vm.state.value.pipYf * freeH
+
+        Box(
+            modifier = Modifier
+                .offset(x = (left / density.density).dp, y = (topPx / density.density).dp)
+                .size(wDp.dp, hDp.dp)
+                .border(2.dp, Color.White, RoundedCornerShape(8.dp))
+                .clip(RoundedCornerShape(8.dp))
+                .pointerInput(state.pipSizeIndex) {
+                    detectDragGestures { change, drag ->
+                        change.consume()
+                        val s = vm.state.value
+                        val nx = (s.pipXf + drag.x / (freeW - rightMargin).coerceAtLeast(1f)).coerceIn(0f, 1f)
+                        val ny = (s.pipYf + drag.y / freeH).coerceIn(0f, 1f)
+                        vm.setPipPos(nx, ny)
+                    }
+                }
+                .pointerInput(state.pipSizeIndex) {
+                    detectTapGestures(
+                        onTap = { vm.cyclePipSize() },
+                        onDoubleTap = { vm.togglePip() },
+                    )
+                },
+        ) {
+            AndroidView(
+                factory = { ctx ->
+                    TextureView(ctx).also { tv ->
+                        tv.surfaceTextureListener = engine
+                        engine.attach(tv)
+                    }
+                },
+                modifier = Modifier.fillMaxSize(),
+            )
+        }
+    }
+}
+
 private fun statusText(state: LiveViewModel.LiveState): String = when (state.status) {
    "no_device" -> "未检测到热像仪，请插入MAG160C"
    "no_permission" -> "USB权限未授予"
@@ -27,6 +27,14 @@ class LiveViewModel(app: Application) : AndroidViewModel(app) {
        val identity: IrSession.CameraIdentity? = null,
        val maxTraceOn: Boolean = true,
        val probes: List<ProbePoint> = emptyList(),
+        /** Visible-light PIP overlay (Phase E). */
+        val pipOn: Boolean = false,
+        /** 0/1/2 -> 96/128/160 dp wide. */
+        val pipSizeIndex: Int = 1,
+        /** PIP top-left X as a fraction of the free space (0..1). */
+        val pipXf: Float = 1f,
+        /** PIP top-left Y as a fraction of the free space below the top bar (0..1). */
+        val pipYf: Float = 0f,
    )

    private val _state = MutableStateFlow(LiveState())
@@ -39,6 +47,14 @@ class LiveViewModel(app: Application) : AndroidViewModel(app) {

    private val session = IrSession(app)

+    companion object {
+        /** PIP overlay width in dp for size index 0/1/2. */
+        val PIP_WIDTHS_DP = intArrayOf(96, 128, 160)
+
+        /** Color-bar footprint kept clear at the right edge (20dp bar + 2x12dp). */
+        const val PIP_RIGHT_MARGIN_DP = 32
+    }
+
    private var usbReceiver: android.content.BroadcastReceiver? = null

    private var recorder: com.mag160c.thermal.media.Mp4Recorder? = null
@@ -200,6 +216,25 @@ class LiveViewModel(app: Application) : AndroidViewModel(app) {
        _state.value = _state.value.copy(maxTraceOn = !_state.value.maxTraceOn)
    }

+    // ---- visible-light PIP (Phase E) ----
+
+    /** PIP width options in dp (matches [PIP_WIDTHS_DP]). */
+    fun togglePip() {
+        _state.value = _state.value.copy(pipOn = !_state.value.pipOn)
+    }
+
+    fun cyclePipSize() {
+        _state.value = _state.value.copy(pipSizeIndex = (_state.value.pipSizeIndex + 1) % 3)
+    }
+
+    /** Drag position as fractions of the free space (clamped to 0..1). */
+    fun setPipPos(xf: Float, yf: Float) {
+        _state.value = _state.value.copy(
+            pipXf = xf.coerceIn(0f, 1f),
+            pipYf = yf.coerceIn(0f, 1f),
+        )
+    }
+
    /** Capture: rendered JPEG + raw frame + camera info -> MDT -> MediaStore. */
    fun capturePhoto(context: android.content.Context) {
        val frame = latestFrame ?: return
@@ -0,0 +1,195 @@
+package com.mag160c.thermal.ui.live
+
+import android.Manifest
+import android.content.Context
+import android.content.pm.PackageManager
+import android.graphics.SurfaceTexture
+import android.hardware.camera2.CameraCaptureSession
+import android.hardware.camera2.CameraCharacteristics
+import android.hardware.camera2.CameraDevice
+import android.hardware.camera2.CameraManager
+import android.hardware.camera2.CaptureRequest
+import android.os.Handler
+import android.os.HandlerThread
+import android.view.Surface
+import android.view.TextureView
+import com.mag160c.thermal.media.DebugLog
+
+/**
+ * Minimal Camera2 preview engine for the visible-light PIP overlay (Phase E).
+ *
+ * Design rules:
+ *  - open the first BACK-facing camera, preview-only (TEMPLATE_PREVIEW), no
+ *    capture requests, no recording, no image reader;
+ *  - every failure path logs under tag "pip" and tears the camera down —
+ *    a camera problem must never crash or block the thermal live view;
+ *  - [release] is idempotent and safe to call from any state.
+ */
+class PipCameraEngine(private val context: Context) : TextureView.SurfaceTextureListener {
+
+    private var textureView: TextureView? = null
+    private var camera: CameraDevice? = null
+    private var session: CameraCaptureSession? = null
+    private var thread: HandlerThread? = null
+    private var handler: Handler? = null
+    private var surface: Surface? = null
+
+    @Volatile
+    private var released = false
+
+    /** True once a repeating preview request has been submitted. */
+    @Volatile
+    var previewing = false
+        private set
+
+    /** Attach to a TextureView and start the preview as soon as it has a surface. */
+    fun attach(view: TextureView) {
+        try {
+            textureView = view
+            view.surfaceTextureListener = this
+            if (view.isAvailable) onSurfaceTextureAvailable(view.surfaceTexture!!, view.width, view.height)
+        } catch (e: Exception) {
+            DebugLog.log("pip", "attach failed: $e")
+        }
+    }
+
+    override fun onSurfaceTextureAvailable(st: SurfaceTexture, width: Int, height: Int) {
+        try {
+            startThread()
+            surface = Surface(st)
+            openCamera()
+        } catch (e: Exception) {
+            DebugLog.log("pip", "surface available handling failed: $e")
+            release()
+        }
+    }
+
+    override fun onSurfaceTextureSizeChanged(st: SurfaceTexture, width: Int, height: Int) = Unit
+
+    override fun onSurfaceTextureDestroyed(st: SurfaceTexture): Boolean {
+        release()
+        return true
+    }
+
+    override fun onSurfaceTextureUpdated(st: SurfaceTexture) = Unit
+
+    private fun startThread() {
+        if (thread == null) {
+            val t = HandlerThread("pip-camera")
+            t.start()
+            thread = t
+            handler = Handler(t.looper)
+        }
+    }
+
+    private fun openCamera() {
+        val manager = context.getSystemService(Context.CAMERA_SERVICE) as CameraManager
+        val id = try {
+            manager.cameraIdList.firstOrNull { cid ->
+                val facing = manager.getCameraCharacteristics(cid)
+                    .get(CameraCharacteristics.LENS_FACING)
+                facing == CameraCharacteristics.LENS_FACING_BACK
+            } ?: manager.cameraIdList.firstOrNull()
+        } catch (e: Exception) {
+            DebugLog.log("pip", "no camera listed: $e")
+            null
+        }
+        if (id == null) {
+            DebugLog.log("pip", "no camera available")
+            return
+        }
+        if (context.checkSelfPermission(Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) {
+            DebugLog.log("pip", "camera permission not granted")
+            return
+        }
+        try {
+            manager.openCamera(id, object : CameraDevice.StateCallback() {
+                override fun onOpened(device: CameraDevice) {
+                    camera = device
+                    DebugLog.log("pip", "camera opened id=$id")
+                    createSession(device)
+                }
+
+                override fun onDisconnected(device: CameraDevice) {
+                    DebugLog.log("pip", "camera disconnected")
+                    release()
+                }
+
+                override fun onError(device: CameraDevice, error: Int) {
+                    DebugLog.log("pip", "camera error=$error")
+                    release()
+                }
+            }, handler)
+        } catch (e: Exception) {
+            DebugLog.log("pip", "openCamera failed: $e")
+            release()
+        }
+    }
+
+    private fun createSession(device: CameraDevice) {
+        val target = surface ?: run {
+            DebugLog.log("pip", "no surface for session")
+            return
+        }
+        try {
+            @Suppress("DEPRECATION")
+            device.createCaptureSession(listOf(target), object : CameraCaptureSession.StateCallback() {
+                override fun onConfigured(s: CameraCaptureSession) {
+                    session = s
+                    startPreview(device, s, target)
+                }
+
+                override fun onConfigureFailed(s: CameraCaptureSession) {
+                    DebugLog.log("pip", "capture session configure failed")
+                    release()
+                }
+            }, handler)
+        } catch (e: Exception) {
+            DebugLog.log("pip", "createCaptureSession failed: $e")
+            release()
+        }
+    }
+
+    private fun startPreview(device: CameraDevice, s: CameraCaptureSession, target: Surface) {
+        try {
+            val request = device.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW).apply {
+                addTarget(target)
+                set(CaptureRequest.CONTROL_MODE, CaptureRequest.CONTROL_MODE_AUTO)
+            }
+            s.setRepeatingRequest(request.build(), null, handler)
+            previewing = true
+            DebugLog.log("pip", "repeating preview started")
+        } catch (e: Exception) {
+            DebugLog.log("pip", "startPreview failed: $e")
+            release()
+        }
+    }
+
+    /** Idempotent teardown: session -> camera -> surface -> thread. */
+    fun release() {
+        if (released && camera == null && session == null) return
+        released = true
+        previewing = false
+        try {
+            session?.close()
+        } catch (e: Exception) {
+            DebugLog.log("pip", "session close failed: $e")
+        }
+        session = null
+        try {
+            camera?.close()
+        } catch (e: Exception) {
+            DebugLog.log("pip", "camera close failed: $e")
+        }
+        camera = null
+        try {
+            surface?.release()
+        } catch (_: Exception) {
+        }
+        surface = null
+        thread?.quitSafely()
+        thread = null
+        handler = null
+        DebugLog.log("pip", "released")
+    }
+}
@@ -0,0 +1,4 @@
+<vector xmlns:android="http://schemas.android.com/apk/res/android" android:width="24dp" android:height="24dp" android:viewportWidth="24" android:viewportHeight="24">
+  <path android:pathData="M12,4a8,8 0 1,0 0.01,0 a8,8 0 1,0 -0.01,0z" android:strokeColor="#FF000000" android:strokeWidth="2" android:fillColor="#00000000" />
+  <path android:pathData="M13,13 h6 v6 h-6 z" android:fillColor="#FF000000" />
+</vector>
@@ -24,6 +24,18 @@
 | 10 | 切到"相册"页 | （无 DebugLog；纯 UI） | 列表出现步骤 8 的照片缩略图（MDT 尾部校验通过才显示） |
 | 11 | 点该照片进入分析页 | （无 DebugLog；纯 UI） | 图片可缩放/平移；调色板重渲染可用；**图像顶部出现温度条**（`中心 x.x℃   最低 x.x℃   最高 x.x℃`）；点图任意位置出现白色圆点+温度标签，再点同一点可清除；备注可编辑保存 |
 | 12 | 分析页生成 PDF 报告 | （无 DebugLog；纯 UI） | 报告文件在 DCIM/MAG160C 或 Download/MAG160C 生成，可打开 |
+| 13 | 实时页顶栏第 5 项点"画中画"（首次会弹相机权限，点允许） | `[pip] camera opened id=<id>` → `[pip] repeating preview started` | 右上角（色标条左侧）出现 128×96dp 白框小窗，显示可见光画面；顶栏该项高亮为"画中画·开" |
+| 14 | 拖动 PIP 小窗到其它角落；单击小窗（循环 96/128/160dp）；双击小窗关闭 | `[pip] released`（双击关闭时） | 小窗跟手移动且不出界；单击在三档宽度间循环；双击后小窗消失，顶栏恢复"画中画"未开启态；日志有 `released` |
+| 15 | PIP 开启时按 Home 回到桌面再返回 | `[pip] released`（进入后台时） | 返回后若 PIP 仍为开启态，画面重新出现（重新 open）；无崩溃 |
+
+## 相机（PIP）失败时的表现（设计如此，不算 bug）
+
+PIP 的相机是可选功能，任何相机异常都只记日志并关闭小窗，**不影响热像主画面**：
+
+- 无相机硬件 / 权限被拒 → `[pip] no camera available` / `[pip] camera permission not granted`，
+  小窗空白或未出现；
+- 相机被其它应用占用 → `[pip] camera error=<N>` + `[pip] released`；
+- 上述情况下主画面、拍照、录像、分析全部照常工作。

 ## 失败时的快速定位（沿用第 11 轮起的诊断路径）

@@ -413,7 +413,17 @@
      proguard-rules.pro 新建（-keep cloud.**；原文件缺失但被 build.gradle
      引用）。新增 CloudClientTest（4 项，锁"默认关闭/api() 拒绝"契约）。
      debug + release(R8) 双构建通过，29 单测全绿。APK 已更新。
- [ ] Phase E：可见光 PIP 融合
+- [x] Phase E（2026-09-10）：可见光 PIP 融合：Manifest 加 CAMERA 权限 +
+      camera.any uses-feature（orientation 未动）；LiveState 加
+      pipOn/pipSizeIndex/pipXf/pipYf + togglePip/cyclePipSize/setPipPos；
+      新增 ui/live/PipCameraView.kt（Camera2 最小实现，TextureView 预览，
+      全部异常只记 DebugLog("pip",…) 并 release，绝不崩溃）；顶栏第 5 项
+      "画中画"+ ic_pip.xml（双弧圆+右下实心矩形）；浮层三档 96/128/160dp
+      （高=宽×3/4），位置相对热像视口、右边缘额外留 32dp 色标条位，
+      拖动/单击换档/双击关闭；PIP 关闭、离页、ON_STOP 三处释放相机；
+      运行时权限用 rememberLauncherForActivityResult。真机项已写入
+      real_device_checklist.md（第 13-15 步 + 失败表现）。APK 已更新。
+- [ ] Phase F：网络互连远程预览


 ## 里程碑日志