姿态识别(YOLOv8-pose FP16) + HDMI多画面输出

- pose_detector: 适配 YOLOv8-pose 输出布局 [1,56,8400] NCHW
  实测 int8 模型量化有缺陷(class通道全被截断为0), 改用 FP16 模型
  FP16 模型需 RKNN_TENSOR_FLOAT16 输入 + 手动 fp16->fp32 转换
  验证: bus.jpg 检测3人, 置信度0.87-0.89, 骨架完整
- display_output: 多路画面拼接输出到本机HDMI显示器(X imshow)
  支持1/2/4/6/9/16宫格, 通道标签, FPS角标
- main.cpp: 接入 DisplayOutput, 姿态识别+跌倒检测联动
- config: pose_model_path 指向 yolov8n-pose_rk3588_fp.rknn, hdmi_output 开关

CMakeLists.txt

@@ -6,7 +6,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -Wall -Wno-unused-variable")
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2")
 
-find_package(OpenCV REQUIRED COMPONENTS core imgproc imgcodecs videoio)
+find_package(OpenCV REQUIRED COMPONENTS core imgproc imgcodecs videoio highgui)
 find_package(Threads REQUIRED)
 
 include_directories(
@@ -35,6 +35,7 @@ add_executable(zone_alarm
     src/bucket_manager.cpp
     src/idle_detector.cpp
     src/pose_detector.cpp
+    src/display_output.cpp
 )
 
 target_link_libraries(zone_alarm

config/config.json

@@ -57,8 +57,8 @@
     "model_path": "models/yolov5s_rk3588_int8.rknn",
     "module_type": "RFID",
     "nms_thresh": 0.45,
-    "pose_enabled": false,
-    "pose_model_path": "models/yolov5s-pose_rk3588_int8.rknn",
+    "pose_enabled": true,
+    "pose_model_path": "models/yolov8n-pose_rk3588_fp.rknn",
     "pose_input_size": 640,
     "pose_conf_thresh": 0.35,
     "pose_nms_thresh": 0.45,
@@ -70,6 +70,8 @@
     "fall_relays": [
         "y01"
     ],
+    "hdmi_output": true,
+    "hdmi_grid_cols": 0,
     "relay": {
         "alarm_max_sec": 30,
         "aliases": {

src/display_output.cpp

@@ -0,0 +1,133 @@
+#include "display_output.h"
+#include <algorithm>
+
+using namespace std;
+
+DisplayOutput::~DisplayOutput() {
+    stop();
+}
+
+bool DisplayOutput::start(bool enable, int grid_cols, const string& title) {
+    enable_ = enable;
+    grid_cols_ = grid_cols;
+    title_ = title;
+    if (!enable_) return false;
+    if (running_.load()) return true;
+
+    running_ = true;
+    worker_ = thread(&DisplayOutput::loop, this);
+    return true;
+}
+
+void DisplayOutput::stop() {
+    if (!running_.load()) return;
+    running_ = false;
+    if (worker_.joinable()) worker_.join();
+    {
+        lock_guard<mutex> lk(mtx_);
+        frames_.clear();
+    }
+}
+
+void DisplayOutput::setFrame(int channel, const cv::Mat& frame,
+                             const string& label) {
+    if (!enable_.load()) return;
+    lock_guard<mutex> lk(mtx_);
+    if (frame.empty()) {
+        frames_.erase(channel);
+        return;
+    }
+    frames_[channel] = {frame.clone(), label};
+}
+
+void DisplayOutput::loop() {
+    int fps_counter = 0;
+    auto last_t = chrono::steady_clock::now();
+    char fps_buf[32];
+
+    while (running_.load()) {
+        auto t0 = chrono::steady_clock::now();
+
+        map<int, pair<cv::Mat, string>> snap;
+        {
+            lock_guard<mutex> lk(mtx_);
+            snap = frames_;
+        }
+
+        if (snap.empty()) {
+            // 无画面: 显示黑色占位
+            cv::Mat black(kWndH, kWndW, CV_8UC3, cv::Scalar(10, 10, 10));
+            cv::putText(black, "ZoneAlarm - waiting for frames...",
+                        cv::Point(50, 50), cv::FONT_HERSHEY_SIMPLEX, 1.0,
+                        cv::Scalar(255, 255, 255), 2);
+            cv::imshow(title_, black);
+        } else {
+            int n = (int)snap.size();
+            int cols = grid_cols_;
+            if (cols <= 0) {
+                // 自动布局: 1->1, 2->2, 3/4->2, 5-6->3, 7-9->3, 10-16->4
+                cols = n <= 1 ? 1 : (n <= 4 ? 2 : (n <= 9 ? 3 : 4));
+            }
+            int rows = (n + cols - 1) / cols;
+
+            int cell_w = kWndW / cols;
+            int cell_h = kWndH / rows;
+            cv::Mat canvas(kWndH, kWndW, CV_8UC3, cv::Scalar(20, 20, 20));
+
+            int idx = 0;
+            for (auto& [ch, p] : snap) {
+                int r = idx / cols, c = idx % cols;
+                int x = c * cell_w, y = r * cell_h;
+                cv::Rect cell(x, y, cell_w, cell_h);
+
+                cv::Mat dst;
+                if (p.first.empty()) {
+                    dst = cv::Mat(cell_h, cell_w, CV_8UC3, cv::Scalar(30, 30, 30));
+                } else {
+                    cv::resize(p.first, dst, cv::Size(cell_w, cell_h));
+                }
+
+                // 边框
+                cv::rectangle(dst, cv::Rect(0, 0, cell_w - 1, cell_h - 1),
+                              cv::Scalar(80, 80, 80), 1);
+
+                // 通道标签
+                string label = p.second.empty()
+                                   ? "CH" + to_string(ch) : p.second;
+                cv::rectangle(dst, cv::Rect(0, 0, cell_w, 26),
+                              cv::Scalar(0, 0, 0), -1);
+                cv::putText(dst, label, cv::Point(6, 19),
+                            cv::FONT_HERSHEY_SIMPLEX, 0.7,
+                            cv::Scalar(255, 255, 255), 1);
+
+                dst.copyTo(canvas(cell));
+                idx++;
+            }
+
+            // 帧率角标
+            auto t1 = chrono::steady_clock::now();
+            fps_counter++;
+            if (chrono::duration_cast<chrono::milliseconds>(t1 - last_t).count() >= 1000) {
+                snprintf(fps_buf, sizeof(fps_buf), "FPS:%d", fps_counter);
+                fps_counter = 0;
+                last_t = t1;
+            }
+            if (fps_buf[0]) {
+                cv::putText(canvas, fps_buf, cv::Point(kWndW - 100, 30),
+                            cv::FONT_HERSHEY_SIMPLEX, 0.7,
+                            cv::Scalar(0, 255, 0), 2);
+            }
+
+            cv::imshow(title_, canvas);
+        }
+
+        cv::waitKey(1);
+
+        // 目标 15 FPS 刷新
+        auto dt = chrono::duration_cast<chrono::milliseconds>(
+                      chrono::steady_clock::now() - t0).count();
+        if (dt < 33) this_thread::sleep_for(chrono::milliseconds(33 - dt));
+    }
+
+    cv::destroyWindow(title_);
+}

src/display_output.h

@@ -0,0 +1,39 @@
+#pragma once
+#include <opencv2/opencv.hpp>
+#include <string>
+#include <vector>
+#include <mutex>
+#include <thread>
+#include <atomic>
+#include <chrono>
+
+// 多路画面拼接输出到本机显示器(HDMI)
+// 支持 1/2/4/6/9/16 宫格布局, 每路左上角标注通道号与状态
+class DisplayOutput {
+public:
+    DisplayOutput() = default;
+    ~DisplayOutput();
+
+    // 初始化: enable=启用输出, grid_cols=网格列数(0=自动)
+    bool start(bool enable, int grid_cols = 0, const std::string& title = "ZoneAlarm");
+    void stop();
+
+    void setFrame(int channel, const cv::Mat& frame, const std::string& label);
+
+    bool isRunning() const { return running_.load(); }
+
+private:
+    void loop();
+
+    std::atomic<bool> running_{false};
+    std::atomic<bool> enable_{false};
+    std::thread worker_;
+    std::string title_ = "ZoneAlarm";
+    int grid_cols_ = 0;
+
+    std::mutex mtx_;
+    std::map<int, std::pair<cv::Mat, std::string>> frames_;
+
+    static constexpr int kWndW = 1920;
+    static constexpr int kWndH = 1080;
+};

src/main.cpp

@@ -38,6 +38,7 @@
 #include "bucket_manager.h"
 #include "idle_detector.h"
 #include "pose_detector.h"
+#include "display_output.h"
 
 using namespace std;
 using json = nlohmann::json;
@@ -297,6 +298,12 @@ int main(int argc, char* argv[]) {
     string fence_file = cfg.value("fence_file", "config/fences.json");
     int alarm_retention_days = cfg.value("alarm_retention_days", 30);
 
+    // ========== 本机画面输出(HDMI显示器) ==========
+    bool hdmi_output = cfg.value("hdmi_output", true);
+    int hdmi_grid_cols = cfg.value("hdmi_grid_cols", 0);
+    cout << "[Display] HDMI画面输出: " << (hdmi_output ? "启用" : "禁用")
+         << " 网格列=" << hdmi_grid_cols << endl;
+
     // ========== 姿态识别(跌倒检测) ==========
     string pose_model_path = cfg.value("pose_model_path",
                                        "models/yolov5s-pose_rk3588_int8.rknn");
@@ -491,6 +498,7 @@ int main(int argc, char* argv[]) {
     vector<shared_ptr<ChannelState>> channels;
 
     WebServer web;
+    DisplayOutput displayOutput;
     web.setPort(web_port);
 
     // 登录凭据(默认admin/admin), 持久化于config.json的web_auth
@@ -1451,6 +1459,7 @@ int main(int argc, char* argv[]) {
     });
 
     web.start();
+    displayOutput.start(hdmi_output, hdmi_grid_cols, "ZoneAlarm - 钢包工位视频监测");
     cout << "[Main] Ctrl+C 退出" << endl;
 
     // 自动激活第一个工位（如果有配置）
@@ -1872,6 +1881,13 @@ int main(int argc, char* argv[]) {
                 ch->display_frame = display;
                 ch->disp_seq.fetch_add(1, std::memory_order_relaxed);
             }
+            // 输出到本机显示器(HDMI)
+            if (displayOutput.isRunning()) {
+                char dlbl[64];
+                snprintf(dlbl, sizeof(dlbl), "CH%d %s",
+                         ch->id, ch->capture->name().c_str());
+                displayOutput.setFrame(ch->id, display, dlbl);
+            }
         }
     }
 

src/pose_detector.cpp

@@ -83,25 +83,58 @@ bool PoseDetector::load(const string& model_path, int input_size,
     return true;
 }
 
+// FP16 <-> FP32 转换(板载 RKNN FP16 模型用)
+static inline uint16_t floatToFp16(float x) {
+    uint32_t f;
+    memcpy(&f, &x, 4);
+    uint32_t sign = (f >> 16) & 0x8000;
+    int32_t e = (int32_t)((f >> 23) & 0xff) - 127 + 15;
+    uint32_t m = f & 0x7fffff;
+    if (((f >> 23) & 0xff) == 0xff) return sign | 0x7c00;   // inf/nan
+    if (e >= 31) return sign | 0x7c00;                       // 溢出
+    if (e <= 0) {
+        if (e < -10) return sign;                            // 下溢为0
+        m = (m | 0x800000) >> (1 - e);
+        return sign | m;
+    }
+    return sign | ((uint32_t)e << 10) | (m >> 13);
+}
+static inline float fp16ToFloat(uint16_t h) {
+    uint32_t sign = ((uint32_t)h & 0x8000) << 16;
+    uint32_t e = (h >> 10) & 0x1f, m = h & 0x3ff;
+    uint32_t f;
+    if (e == 0) {
+        if (m == 0) { f = sign; }
+        else { e = 1; m = (m << 1) + 1; f = sign | ((e + 112) << 23) | (m << 13); }
+    } else if (e == 31) {
+        f = sign | 0x7f800000 | (m << 13);
+    } else {
+        f = sign | ((e + 112) << 23) | (m << 13);
+    }
+    float r;
+    memcpy(&r, &f, 4);
+    return r;
+}
+
 vector<PersonPose> PoseDetector::detect(const float* rgb_data, int img_w, int img_h,
                                         float scale, int pad_x, int pad_y) {
     if (!loaded_ || !rgb_data) return {};
 
     int num_pixels = input_size_ * input_size_ * 3;
 
-    vector<int8_t> input_i8(num_pixels);
+    // FP16 输入: 直接转 float32 rgb [0,1] -> fp16
+    vector<uint16_t> input_fp16(num_pixels);
     for (int i = 0; i < num_pixels; i++) {
-        int v = (int)(rgb_data[i] * 255.0f) - 128;
-        input_i8[i] = (int8_t)(v < -128 ? -128 : (v > 127 ? 127 : v));
+        input_fp16[i] = floatToFp16(rgb_data[i]);
     }
 
     rknn_input inputs[1];
     memset(&inputs[0], 0, sizeof(rknn_input));
     inputs[0].index = 0;
-    inputs[0].type = RKNN_TENSOR_INT8;
+    inputs[0].type = RKNN_TENSOR_FLOAT16;
     inputs[0].fmt = RKNN_TENSOR_NHWC;
-    inputs[0].size = input_i8.size() * sizeof(int8_t);
-    inputs[0].buf = input_i8.data();
+    inputs[0].size = input_fp16.size() * sizeof(uint16_t);
+    inputs[0].buf = input_fp16.data();
     inputs[0].pass_through = 1;
 
     int ret = rknn_inputs_set(ctx_, 1, inputs);
@@ -119,7 +152,7 @@ vector<PersonPose> PoseDetector::detect(const float* rgb_data, int img_w, int im
     vector<rknn_output> outputs(n_output_);
     for (uint32_t i = 0; i < n_output_; i++) {
         memset(&outputs[i], 0, sizeof(rknn_output));
-        outputs[i].want_float = 1;
+        outputs[i].want_float = 0;   // FP16 模型: 拿原始 fp16 数据自己转
     }
     ret = rknn_outputs_get(ctx_, n_output_, outputs.data(), nullptr);
     if (ret < 0) {
@@ -131,36 +164,20 @@ vector<PersonPose> PoseDetector::detect(const float* rgb_data, int img_w, int im
 
     if (n_output_ == 1) {
         auto& attr = output_attrs_[0];
-        int num_preds = attr.dims[1];
-        int num_attrs = attr.dims[2];
-        float* data = static_cast<float*>(outputs[0].buf);
-
-        // YOLOv8-pose 输出: num_preds x (4 + 1 + num_cls + 17*3)
-        // 其中 17*3 = 每个关键点 (x, y, conf)
-        int num_cls = num_attrs - 4 - 1 - 17 * 3;
-        if (num_cls >= 1) {
+        // RKNN FP16 输出: [1, 56, 8400] NCHW (通道在前), 原始 fp16
+        // YOLOv8-pose: 56 = 4(bbox cx,cy,w,h) + 1(class) + 17*3(kpt x,y,conf)
+        int ch = attr.dims[1];
+        int num_preds = attr.dims[2];
+        uint16_t* data = static_cast<uint16_t*>(outputs[0].buf);
+        if (ch >= 4 + 1 + 17 * 3 && num_preds > 0) {
             for (int i = 0; i < num_preds; i++) {
-                float* pred = data + i * num_attrs;
-
-                float box_conf = pred[4];
-                float max_score = 0;
-                int max_id = -1;
-                for (int c = 5; c < 5 + num_cls; c++) {
-                    if (pred[c] > max_score) {
-                        max_score = pred[c];
-                        max_id = c - 5;
-                    }
-                }
-                // 类别0(person)之外的只当背景处理
-                if (max_id != 0) continue;
-
-                float final_score = box_conf * max_score;
-                if (final_score < conf_thresh_) continue;
+                float cx = fp16ToFloat(data[0 * num_preds + i]);
+                float cy = fp16ToFloat(data[1 * num_preds + i]);
+                float w  = fp16ToFloat(data[2 * num_preds + i]);
+                float h  = fp16ToFloat(data[3 * num_preds + i]);
 
-                float cx = pred[0];
-                float cy = pred[1];
-                float w = pred[2];
-                float h = pred[3];
+                float cls = fp16ToFloat(data[4 * num_preds + i]);
+                if (cls < conf_thresh_) continue;
 
                 float x1 = (cx - w * 0.5f - pad_x) / scale;
                 float y1 = (cy - h * 0.5f - pad_y) / scale;
@@ -177,16 +194,16 @@ vector<PersonPose> PoseDetector::detect(const float* rgb_data, int img_w, int im
 
                 PersonPose pp;
                 pp.bbox = cv::Rect(ix1, iy1, bw, bh);
-                pp.confidence = final_score;
+                pp.confidence = cls;
 
-                float* kp = pred + 5 + num_cls;
                 pp.keypoints.resize(17);
                 pp.keypoint_conf.resize(17);
                 for (int k = 0; k < 17; k++) {
-                    float kx = (kp[k * 3] - pad_x) / scale;
-                    float ky = (kp[k * 3 + 1] - pad_y) / scale;
+                    int base = 5 + k * 3;
+                    float kx = (fp16ToFloat(data[(base + 0) * num_preds + i]) - pad_x) / scale;
+                    float ky = (fp16ToFloat(data[(base + 1) * num_preds + i]) - pad_y) / scale;
                     pp.keypoints[k] = cv::Point2f(kx, ky);
-                    pp.keypoint_conf[k] = kp[k * 3 + 2];
+                    pp.keypoint_conf[k] = fp16ToFloat(data[(base + 2) * num_preds + i]);
                 }
                 poses.push_back(std::move(pp));
             }