OPENCV——ROCKX+RV1126实现1-＞N人脸识别功能

一、人脸识别功能大体流程上图是实现1-N人脸识别流程首先要初始化RV1126模块初始化包括VI模块、VENC模块、人脸检测rockx模块、人脸识别rockx模块初始化模块之后就要分两个线程做处理。主流程是先读取单张图片的图像并提取人脸特征值然后死循环获取VI的码流数据然后用rockx的人脸检测模块RV1126的VI数据是否有人脸如果有人脸则调用rockx的人脸识别模块识别出RV1126视频流的人脸数据并且提取出来。然后对比两个人脸的阈值如果1.0则认定单张人脸图片和RV1126检测的人脸是同一个否则就不是同一个人并把数据通过Opencv显示到VI数据最后把识别后的VI数据传输到VENC编码器里面get_rockx_face_recg_venc_thread线程主要是获取每一帧的VENC码流数据并且保存起来。二、代码#include assert.h #include fcntl.h #include getopt.h #include opencv2/core/hal/interface.h #include opencv2/imgproc.hpp //#include opencv2/imgproc/imgproc_c.h #include pthread.h #include signal.h #include stdbool.h #include stdio.h #include stdlib.h #include time.h #include unistd.h #include string // #include common/sample_common.h #include rkmedia_api.h #include rockx.h #include opencv2/core.hpp // #include opencv2/imgoroc.hpp #include opencv2/highgui.hpp #include opencv2/opencv.hpp #define CAMERA_PATH rkispp_scale0 #define CAMERA_ID 0 #define CAMERA_CHN 0 #define VENC_CHN 0 #define WIDTH 1920 #define HEIGHT 1080 using namespace std; using namespace cv; string reco_name; void * get_rockx_face_recg_venc_thread(void * args) { pthread_detach(pthread_self()); FILE * face_reco_venc_h264 fopen(face_reco.h264, w); MEDIA_BUFFER mb NULL; while(1) { mb RK_MPI_SYS_GetMediaBuffer(RK_ID_VENC, VENC_CHN, -1); if(!mb) { printf(Get Venc Stream break...\n); break; } fwrite(RK_MPI_MB_GetPtr(mb), RK_MPI_MB_GetSize(mb), 1, face_reco_venc_h264); RK_MPI_MB_ReleaseBuffer(mb); } return NULL; } int main(int argc, char **argv) { const char * img_path argv[1]; int ret; VI_CHN_ATTR_S vi_chn_attr; vi_chn_attr.pcVideoNode CAMERA_PATH; // Path vi_chn_attr.u32Width WIDTH; // Width vi_chn_attr.u32Height HEIGHT; // Height vi_chn_attr.enPixFmt IMAGE_TYPE_NV12; // ImageType vi_chn_attr.enBufType VI_CHN_BUF_TYPE_MMAP; // BufType vi_chn_attr.u32BufCnt 3; // Cnt vi_chn_attr.enWorkMode VI_WORK_MODE_NORMAL; // Mode ret RK_MPI_VI_SetChnAttr(CAMERA_ID, CAMERA_CHN, vi_chn_attr); if (ret) { printf(Vi Set Attr Failed.....\n); return 0; } else { printf(Vi Set Attr Success.....\n); } ret RK_MPI_VI_EnableChn(CAMERA_ID, CAMERA_CHN); if (ret) { printf(Vi Enable Attr Failed.....\n); return 0; } else { printf(Vi Enable Attr Success.....\n); } VENC_CHN_ATTR_S venc_chn_attr; memset(venc_chn_attr, 0, sizeof(VENC_CHN_ATTR_S)); venc_chn_attr.stVencAttr.u32PicWidth WIDTH; venc_chn_attr.stVencAttr.u32PicHeight HEIGHT; venc_chn_attr.stVencAttr.u32VirWidth WIDTH; venc_chn_attr.stVencAttr.u32VirHeight HEIGHT; venc_chn_attr.stVencAttr.imageType IMAGE_TYPE_NV12; venc_chn_attr.stVencAttr.enType RK_CODEC_TYPE_H264; venc_chn_attr.stVencAttr.u32Profile 66; venc_chn_attr.stRcAttr.enRcMode VENC_RC_MODE_H264CBR; venc_chn_attr.stRcAttr.stH264Cbr.u32Gop 25; venc_chn_attr.stRcAttr.stH264Cbr.u32BitRate WIDTH * HEIGHT * 3; venc_chn_attr.stRcAttr.stH264Cbr.fr32DstFrameRateDen 1; venc_chn_attr.stRcAttr.stH264Cbr.fr32DstFrameRateNum 25; venc_chn_attr.stRcAttr.stH264Cbr.u32SrcFrameRateDen 1; venc_chn_attr.stRcAttr.stH264Cbr.u32SrcFrameRateNum 25; ret RK_MPI_VENC_CreateChn(VENC_CHN, venc_chn_attr); if (ret) { printf(ERROR: Create venc failed!\n); exit(0); } ret RK_MPI_VI_StartStream(CAMERA_ID, CAMERA_CHN); if (ret) { printf(ERROR: RK_MPI_VI_StartStream failed!\n); exit(0); } rockx_config_t * face_config rockx_create_config(); rockx_add_config(face_config, ROCKX_CONFIG_DATA_PATH, /userdata/rockx_data); rockx_handle_t face_det_handle; rockx_module_t face_det_module ROCKX_MODULE_FACE_DETECTION_V2; rockx_ret_t face_det_ret; face_det_ret rockx_create(face_det_handle, face_det_module, face_config, 0); if(face_det_ret ! ROCKX_RET_SUCCESS) { printf(rockx_create face_detect_handle failed!\n); return -1; } rockx_handle_t face_reco_handle; rockx_module_t face_reco_module ROCKX_MODULE_FACE_RECOGNIZE; rockx_ret_t face_recg_ret; face_recg_ret rockx_create(face_reco_handle, face_reco_module, face_config, 0); if(face_recg_ret ! ROCKX_RET_SUCCESS) { printf(rockx_create face_recognize_handle failed!\n); return -1; } rockx_image_t single_face_image; rockx_image_read(img_path, single_face_image, 1); rockx_face_feature_t single_face_feature; rockx_face_recognize(face_reco_handle, single_face_image, single_face_feature); pthread_t pid; pthread_create(pid, NULL, get_rockx_face_recg_venc_thread, NULL); MEDIA_BUFFER mb; rockx_image_t rv1126_video_image; rv1126_video_image.width WIDTH; rv1126_video_image.height HEIGHT; rv1126_video_image.pixel_format ROCKX_PIXEL_FORMAT_YUV420SP_NV12; rockx_ret_t sim_ret; Point text_point; text_point.x 300; text_point.y 300; while(1) { mb RK_MPI_SYS_GetMediaBuffer(RK_ID_VI, CAMERA_CHN,-1); if(!mb) { printf(Get Vi_data break....\n); break; } rv1126_video_image.data (uint8_t *)RK_MPI_MB_GetPtr(mb); rv1126_video_image.size RK_MPI_MB_GetSize(mb); Mat rv1126_mat Mat(HEIGHT, WIDTH,CV_8UC1, rv1126_video_image.data); rockx_object_array_t face_detect_array; face_det_ret rockx_face_detect(face_det_handle, rv1126_video_image, face_detect_array, NULL); if(face_det_ret ! ROCKX_RET_SUCCESS) { printf(rockx_face_detect failed...\n); } rockx_face_feature_t rv1126_face_feature; if(face_detect_array.count 0) { face_recg_ret rockx_face_recognize(face_reco_handle, rv1126_video_image, rv1126_face_feature); if(face_recg_ret ! ROCKX_RET_SUCCESS) { printf(rockx_face_recognize faiked...\n); } float similarity; sim_ret rockx_face_feature_similarity(single_face_feature, rv1126_face_feature, similarity); if(sim_ret ! ROCKX_RET_SUCCESS) { printf(rockx_face_feature_similarity failed ...\n); } if(similarity 1.0) { reco_name 123; printf(This is 123...\n); } else { reco_name ; printf(Can not recognize...\n); } putText(rv1126_mat, reco_name, text_point, FONT_HERSHEY_COMPLEX, 1.0, Scalar(255,0,255), 1); } RK_MPI_SYS_SendMediaBuffer(RK_ID_VENC, VENC_CHN, mb); RK_MPI_MB_ReleaseBuffer(mb); } RK_MPI_VI_DisableChn(CAMERA_ID, CAMERA_CHN); RK_MPI_VENC_DestroyChn(VENC_CHN); return 0; }2.1. RV1126模块的初始化并启动VI工作初始化RV1126的VI模块、VENC模块、并调用RK_MPI_VI_StartStream启动VI模块采集摄像头数据。具体的API不讲解了因为之前已经讲了很多次。2.2. 初始化人脸检测和人脸识别的rockx模块rockx_config_t * face_config rockx_create_config(); rockx_add_config(face_config, ROCKX_CONFIG_DATA_PATH, /userdata/rockx_data);上面是rockx模块的初始化这里我们要初始化两个rockx模块分别是人脸检测rockx模块和人脸识别rockx模块。首先要使用rockx_create_config分配rockx_config_t结构体并使用rockx_add_config把对应的rockx路径配置进去在我们的板子里面在/userdata/rockx_data里面。rockx_handle_t face_det_handle; rockx_module_t face_det_module ROCKX_MODULE_FACE_DETECTION_V2; rockx_ret_t face_det_ret; face_det_ret rockx_create(face_det_handle, face_det_module, face_config, 0); if(face_det_ret ! ROCKX_RET_SUCCESS) { printf(rockx_create face_detect_handle failed!\n); return -1; }使用rockx_create创建人脸检测rockx_handle_t句柄。rockx_create的传参第一个参数rockx_handle_t结构体指针、 第二个参数rockx_module_t是ROCKX_MODULE_FACE_DETECTION_V2ROCKX_MODULE_FACE_DETECTION_V2是人脸检测的Version2模块、第三个参数是rockx_config_t结构体指针、第四个参数默认是0。rockx_handle_t face_reco_handle; rockx_module_t face_reco_module ROCKX_MODULE_FACE_RECOGNIZE; rockx_ret_t face_recg_ret; face_recg_ret rockx_create(face_reco_handle, face_reco_module, face_config, 0); if(face_recg_ret ! ROCKX_RET_SUCCESS) { printf(rockx_create face_recognize_handle failed!\n); return -1; }使用rockx_create创建人脸识别rockx_handle_t句柄。rockx_create的传参第一个参数rockx_handle_t结构体指针、 第二个参数rockx_module_t是ROCKX_MODULE_FACE_RECOGNIZEROCKX_MODULE_FACE_RECOGNIZE是人脸识别模块、第三个参数是rockx_config_t结构体指针、第四个参数默认是0。2.3. 读取单张人脸的图像并提取特征值rockx_image_t single_face_image; rockx_image_read(img_path, single_face_image, 1); rockx_face_feature_t single_face_feature; rockx_face_recognize(face_reco_handle, single_face_image, single_face_feature);用rockx_image_read读取单张人脸图像face_02.jpg这张图片是我上节课拍照的图片并且用rockx_face_recognize提取单张图片的人脸特征值第一个参数传值人脸识别的句柄(face_reco_handle)第二个参数传值单张人脸的rockx_image_t数据地址(single_face_image)第三个参数传值获取单张人脸的人脸特征(single_face_feature)。2.4. 获取每一帧VI视频数据并提取VI数据的人脸特征值MEDIA_BUFFER mb; rockx_image_t rv1126_video_image; rv1126_video_image.width WIDTH; rv1126_video_image.height HEIGHT; rv1126_video_image.pixel_format ROCKX_PIXEL_FORMAT_YUV420SP_NV12;首先初始化rockx_image_t结构体包括width(WIDTH1920),height(HEIGHT1080),pixel_format(ROCKX_PIXEL_FORMAT_YUV420SP_NV12)。使用RK_MPI_SYS_GetMediaBuffer获取每一帧VI数据并且把每一帧VI数据赋值到rockx_image_t这里关键要赋值的是data(data (uint8_t *)RK_MPI_MB_GetPtr(mb) )和size(size RK_MPI_MB_GetSize(mb))。while(1) { mb RK_MPI_SYS_GetMediaBuffer(RK_ID_VI, CAMERA_CHN,-1); if(!mb) { printf(Get Vi_data break....\n); break; } rv1126_video_image.data (uint8_t *)RK_MPI_MB_GetPtr(mb); rv1126_video_image.size RK_MPI_MB_GetSize(mb); rockx_object_array_t face_detect_array; face_det_ret rockx_face_detect(face_det_handle, rv1126_video_image, face_detect_array, NULL); if(face_det_ret ! ROCKX_RET_SUCCESS) { printf(rockx_face_detect failed...\n); } rockx_face_feature_t rv1126_face_feature; if(face_detect_array.count 0) { face_recg_ret rockx_face_recognize(face_reco_handle, rv1126_video_image, rv1126_face_feature); if(face_recg_ret ! ROCKX_RET_SUCCESS) { printf(rockx_face_recognize faiked...\n); }赋值后就调用rockx_face_detect对每一帧的视频数据进行人脸检测如果rockx_object_array_t的count大于0(face_detect_array.count)则说明检测到人脸。若检测到人脸则调用rockx_face_recognize提取VI摄像头的人脸特征值第一个参数传值人脸识别的句柄(face_reco_handle)第二个参数传值rv1126的图像数据地址(rv1126_video_image)第三个参数传值获取rv1126视频流的人脸特征(rv1126_face_feature)。2.5. 对比单张图片的人脸特征值和VI数据的人脸特征值float similarity; sim_ret rockx_face_feature_similarity(single_face_feature, rv1126_face_feature, similarity); if(sim_ret ! ROCKX_RET_SUCCESS) { printf(rockx_face_feature_similarity failed ...\n); } if(similarity 1.0) { reco_name 123; printf(This is 123...\n); } else { reco_name ; printf(Can not recognize...\n); }提取完单张图片特征值和RV1126视频流的人脸特征值后就要对比两个特征值的相似度了在rockx框架提供了rockx_face_feature_similarity去对比两个脸特征值。第一个参数传值单张人脸的特征结构体指针(single_face_feature), 第二个参数rv1126视频流人脸特征结构体指针(rv1126_face_feature)第三个参数两个人脸对比输出的相似度阈值similarity(similarity)。如果输出的人脸相似度值1.0(1.0是rockx比较经典的人脸识别阈值, 值越小相似度越高)则说明单张人脸和视频流的人脸是同一个人然后把”Harry”输出到string , 否则就不输出。2.6.把识别的人脸名称输出到opencv的TextMat rv1126_mat Mat(HEIGHT, WIDTH,CV_8UC1, rv1126_video_image.data);上述步骤已经得到了对比的人脸名称这一步则需要把人脸的名称输出到OPENCV里面并显示到视频上。首先要创建OPENCV的矩阵Mat rv1126_mat Mat(HEIGHT,WIDTH,CV_8UC1,rv1126_video_image.data),第一个参数HEIGHT(1080),第二个参数HEIGHT(1920),第三个参数CV_8UC1(单通道),第四个参数rv1126_video_image.data(每一帧的rv1126的VI数据)。putText(rv1126_mat, reco_name, text_point, FONT_HERSHEY_COMPLEX, 1.0, Scalar(255,0,255), 1);创建完矩阵后则需要调用OPENCV的putText把人脸名称输出到Mat矩阵里面。具体的如下putText(rv1126_mat,reco_name,text_point,FONT_HERSHEY_COMPLEX,1.0,Scalar(255,0,255),1)第一个参数rv1126_mat(Mat矩阵数据)第二个参数reco_name(人脸名称的string字符串)第三个参数text_point(坐标信息,x轴300,y轴-300),第四个参数FONT_HERSHEY_COMPLEX(字体类型)第五个参数1.0(字体大小是1.0)第六个参数Scalar(255,0,255)(颜色标量)第七个参数1(粗细程度是1)。2.7.把处理后的数据传输到VENC编码器RK_MPI_SYS_SendMediaBuffer(RK_ID_VENC, VENC_CHN, mb); RK_MPI_MB_ReleaseBuffer(mb);上述识别工作完成之后就需要把VI数据传输到VENC编码器里面。这里直接调用RK_MPI_SYS_SendMediaBuffer(RK_ID_VENC, VENC_CHN, mb)去发送。2.8.开启get_rockx_face_recg_venc_thread线程获取每一帧VENC数据并且保存到H264pthread_t pid; pthread_create(pid, NULL, get_rockx_face_recg_venc_thread, NULL);创建一个get_rockx_face_recg_venc_thread线程void * get_rockx_face_recg_venc_thread(void * args) { pthread_detach(pthread_self()); FILE * face_reco_venc_h264 fopen(face_reco.h264, w); MEDIA_BUFFER mb NULL; while(1) { mb RK_MPI_SYS_GetMediaBuffer(RK_ID_VENC, VENC_CHN, -1); if(!mb) { printf(Get Venc Stream break...\n); break; } fwrite(RK_MPI_MB_GetPtr(mb), RK_MPI_MB_GetSize(mb), 1, face_reco_venc_h264); RK_MPI_MB_ReleaseBuffer(mb); } return NULL; }在这个线程里面通过RK_MPI_SYS_GetMediaBuffer(RK_ID_VENC, VENC_CHN, -1)获取每一帧VENC码流并且用fwrite写到face_reco.h264文件里面。程序运行的效果在这个H264里面可以识别到人脸名称并且显示出来。这个程序只是能跑通整个流程细节方面还有很多问题比如说应该不给人脸识别模块喂整帧画面最好是提前画框将人脸区域裁剪出来再喂给人脸识别模块。每一帧都识别对板子的npu占用太大最好做到跳帧识别最好加上正确的退出逻辑。