前言
1.關于證件照，有好多種制作辦法，最常見的是使用PS來做影像處理，或者下載各種證件照相關的APP，一鍵制作，基本的步驟是先按人臉為基準切出適合的尺寸，然后把人像給摳出來，對人像進行美化處理，然后替換上要使用的背景色，比如藍色或紅色，
2.我這里也按著上面的步驟來用代碼實作，先是人臉檢測，剪切照片，替換背景色，美化和修臉暫時還沒有時間寫完，
3.因為是考慮到要移植到移動端（安卓和iOS），這里使用了ncnn做推理加速庫，之前做過一些APP，加速庫都選了ncnn，不管在安卓或者iOS上,性能都是不錯的，
4.我的開發環境是win10， vs2019, opencv4.5, ncnn,如果要啟用GPU加速，所以用到VulkanSDK，實作語言是C++，
5.先上效果圖，對于背景純度的要求不高，如果使用場景背景復雜的話，也可以完美摳圖，
原始影像：
原圖：
自動剪切出來的證件照：
原圖：
自動剪切出來的證件照：
一.專案創建
1.使用vs2019新建一個C++專案，把OpenC和NCNN庫匯入，NCNN可以下載官方編譯好的庫，我也會在后面上傳我使用的庫和原始碼以及用到的模型，
2.如果要啟用GPU推理，就要安裝VulkanSDK，安裝的步驟可以參考我之前的博客，
二.人臉檢測
1.人臉檢測這里面使用 SCRFD ，它帶眼睛，鼻子，嘴角五個關鍵點的坐標，這個可以用做證件照參考點，人臉檢測庫這個也可以用libfacedetection，效果都差不多，如果是移動端最好選擇SCRFD，
代碼實作：
推理代碼
#include "scrfd.h"
#include <string.h>
#include <opencv2/core/core.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <ncnn/cpu.h> //安卓才用到
static inline float intersection_area(const FaceObject& a, const FaceObject& b)
{
cv::Rect_<float> inter = a.rect & b.rect;
return inter.area();
}
static void qsort_descent_inplace(std::vector<FaceObject>& faceobjects, int left, int right)
{
int i = left;
int j = right;
float p = faceobjects[(left + right) / 2].prob;
while (i <= j)
{
while (faceobjects[i].prob > p)
i++;
while (faceobjects[j].prob < p)
j--;
if (i <= j)
{
// swap
std::swap(faceobjects[i], faceobjects[j]);
i++;
j--;
}
}
// #pragma omp parallel sections
{
// #pragma omp section
{
if (left < j) qsort_descent_inplace(faceobjects, left, j);
}
// #pragma omp section
{
if (i < right) qsort_descent_inplace(faceobjects, i, right);
}
}
}
static void qsort_descent_inplace(std::vector<FaceObject>& faceobjects)
{
if (faceobjects.empty())
return;
qsort_descent_inplace(faceobjects, 0, faceobjects.size() - 1);
}
static void nms_sorted_bboxes(const std::vector<FaceObject>& faceobjects, std::vector<int>& picked, float nms_threshold)
{
picked.clear();
const int n = faceobjects.size();
std::vector<float> areas(n);
for (int i = 0; i < n; i++)
{
areas[i] = faceobjects[i].rect.area();
}
for (int i = 0; i < n; i++)
{
const FaceObject& a = faceobjects[i];
int keep = 1;
for (int j = 0; j < (int)picked.size(); j++)
{
const FaceObject& b = faceobjects[picked[j]];
// intersection over union
float inter_area = intersection_area(a, b);
float union_area = areas[i] + areas[picked[j]] - inter_area;
// float IoU = inter_area / union_area
if (inter_area / union_area > nms_threshold)
keep = 0;
}
if (keep)
picked.push_back(i);
}
}
static ncnn::Mat generate_anchors(int base_size, const ncnn::Mat& ratios, const ncnn::Mat& scales)
{
int num_ratio = ratios.w;
int num_scale = scales.w;
ncnn::Mat anchors;
anchors.create(4, num_ratio * num_scale);
const float cx = 0;
const float cy = 0;
for (int i = 0; i < num_ratio; i++)
{
float ar = ratios[i];
int r_w = round(base_size / sqrt(ar));
int r_h = round(r_w * ar); //round(base_size * sqrt(ar));
for (int j = 0; j < num_scale; j++)
{
float scale = scales[j];
float rs_w = r_w * scale;
float rs_h = r_h * scale;
float* anchor = anchors.row(i * num_scale + j);
anchor[0] = cx - rs_w * 0.5f;
anchor[1] = cy - rs_h * 0.5f;
anchor[2] = cx + rs_w * 0.5f;
anchor[3] = cy + rs_h * 0.5f;
}
}
return anchors;
}
static void generate_proposals(const ncnn::Mat& anchors, int feat_stride, const ncnn::Mat& score_blob, const ncnn::Mat& bbox_blob, const ncnn::Mat& kps_blob, float prob_threshold, std::vector<FaceObject>& faceobjects)
{
int w = score_blob.w;
int h = score_blob.h;
// generate face proposal from bbox deltas and shifted anchors
const int num_anchors = anchors.h;
for (int q = 0; q < num_anchors; q++)
{
const float* anchor = anchors.row(q);
const ncnn::Mat score = score_blob.channel(q);
const ncnn::Mat bbox = bbox_blob.channel_range(q * 4, 4);
// shifted anchor
float anchor_y = anchor[1];
float anchor_w = anchor[2] - anchor[0];
float anchor_h = anchor[3] - anchor[1];
for (int i = 0; i < h; i++)
{
float anchor_x = anchor[0];
for (int j = 0; j < w; j++)
{
int index = i * w + j;
float prob = score[index];
if (prob >= prob_threshold)
{
// insightface/detection/scrfd/mmdet/models/dense_heads/scrfd_head.py _get_bboxes_single()
float dx = bbox.channel(0)[index] * feat_stride;
float dy = bbox.channel(1)[index] * feat_stride;
float dw = bbox.channel(2)[index] * feat_stride;
float dh = bbox.channel(3)[index] * feat_stride;
// insightface/detection/scrfd/mmdet/core/bbox/transforms.py distance2bbox()
float cx = anchor_x + anchor_w * 0.5f;
float cy = anchor_y + anchor_h * 0.5f;
float x0 = cx - dx;
float y0 = cy - dy;
float x1 = cx + dw;
float y1 = cy + dh;
FaceObject obj;
obj.rect.x = x0;
obj.rect.y = y0;
obj.rect.width = x1 - x0 + 1;
obj.rect.height = y1 - y0 + 1;
obj.prob = prob;
if (!kps_blob.empty())
{
const ncnn::Mat kps = kps_blob.channel_range(q * 10, 10);
obj.landmark[0].x = cx + kps.channel(0)[index] * feat_stride;
obj.landmark[0].y = cy + kps.channel(1)[index] * feat_stride;
obj.landmark[1].x = cx + kps.channel(2)[index] * feat_stride;
obj.landmark[1].y = cy + kps.channel(3)[index] * feat_stride;
obj.landmark[2].x = cx + kps.channel(4)[index] * feat_stride;
obj.landmark[2].y = cy + kps.channel(5)[index] * feat_stride;
obj.landmark[3].x = cx + kps.channel(6)[index] * feat_stride;
obj.landmark[3].y = cy + kps.channel(7)[index] * feat_stride;
obj.landmark[4].x = cx + kps.channel(8)[index] * feat_stride;
obj.landmark[4].y = cy + kps.channel(9)[index] * feat_stride;
}
faceobjects.push_back(obj);
}
anchor_x += feat_stride;
}
anchor_y += feat_stride;
}
}
}
SCRFD::SCRFD()
{}
int SCRFD::detect(const cv::Mat& rgb, std::vector<FaceObject>& faceobjects, float prob_threshold, float nms_threshold)
{
int width = rgb.cols;
int height = rgb.rows;
// insightface/detection/scrfd/configs/scrfd/scrfd_500m.py
const int target_size = 640;
// pad to multiple of 32
int w = width;
int h = height;
float scale = 1.f;
if (w > h)
{
scale = (float)target_size / w;
w = target_size;
h = h * scale;
}
else
{
scale = (float)target_size / h;
h = target_size;
w = w * scale;
}
ncnn::Mat in = ncnn::Mat::from_pixels_resize(rgb.data, ncnn::Mat::PIXEL_RGB, width, height, w, h);
// pad to target_size rectangle
int wpad = (w + 31) / 32 * 32 - w;
int hpad = (h + 31) / 32 * 32 - h;
ncnn::Mat in_pad;
ncnn::copy_make_border(in, in_pad, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, ncnn::BORDER_CONSTANT, 0.f);
const float mean_vals[3] = {127.5f, 127.5f, 127.5f};
const float norm_vals[3] = {1/128.f, 1/128.f, 1/128.f};
in_pad.substract_mean_normalize(mean_vals, norm_vals);
ncnn::Extractor ex = scrfd_net.create_extractor();
ex.input("input.1", in_pad);
std::vector<FaceObject> faceproposals;
// stride 8
{
ncnn::Mat score_blob, bbox_blob, kps_blob;
ex.extract("score_8", score_blob);
ex.extract("bbox_8", bbox_blob);
if (has_kps)
ex.extract("kps_8", kps_blob);
const int base_size = 16;
const int feat_stride = 8;
ncnn::Mat ratios(1);
ratios[0] = 1.f;
ncnn::Mat scales(2);
scales[0] = 1.f;
scales[1] = 2.f;
ncnn::Mat anchors = generate_anchors(base_size, ratios, scales);
std::vector<FaceObject> faceobjects32;
generate_proposals(anchors, feat_stride, score_blob, bbox_blob, kps_blob, prob_threshold, faceobjects32);
faceproposals.insert(faceproposals.end(), faceobjects32.begin(), faceobjects32.end());
}
// stride 16
{
ncnn::Mat score_blob, bbox_blob, kps_blob;
ex.extract("score_16", score_blob);
ex.extract("bbox_16", bbox_blob);
if (has_kps)
ex.extract("kps_16", kps_blob);
const int base_size = 64;
const int feat_stride = 16;
ncnn::Mat ratios(1);
ratios[0] = 1.f;
ncnn::Mat scales(2);
scales[0] = 1.f;
scales[1] = 2.f;
ncnn::Mat anchors = generate_anchors(base_size, ratios, scales);
std::vector<FaceObject> faceobjects16;
generate_proposals(anchors, feat_stride, score_blob, bbox_blob, kps_blob, prob_threshold, faceobjects16);
faceproposals.insert(faceproposals.end(), faceobjects16.begin(), faceobjects16.end());
}
// stride 32
{
ncnn::Mat score_blob, bbox_blob, kps_blob;
ex.extract("score_32", score_blob);
ex.extract("bbox_32", bbox_blob);
if (has_kps)
ex.extract("kps_32", kps_blob);
const int base_size = 256;
const int feat_stride = 32;
ncnn::Mat ratios(1);
ratios[0] = 1.f;
ncnn::Mat scales(2);
scales[0] = 1.f;
scales[1] = 2.f;
ncnn::Mat anchors = generate_anchors(base_size, ratios, scales);
std::vector<FaceObject> faceobjects8;
generate_proposals(anchors, feat_stride, score_blob, bbox_blob, kps_blob, prob_threshold, faceobjects8);
faceproposals.insert(faceproposals.end(), faceobjects8.begin(), faceobjects8.end());
}
// sort all proposals by score from highest to lowest
qsort_descent_inplace(faceproposals);
// apply nms with nms_threshold
std::vector<int> picked;
nms_sorted_bboxes(faceproposals, picked, nms_threshold);
int face_count = picked.size();
faceobjects.resize(face_count);
for (int i = 0; i < face_count; i++)
{
faceobjects[i] = faceproposals[picked[i]];
// adjust offset to original unpadded
float x0 = (faceobjects[i].rect.x - (wpad / 2)) / scale;
float y0 = (faceobjects[i].rect.y - (hpad / 2)) / scale;
float x1 = (faceobjects[i].rect.x + faceobjects[i].rect.width - (wpad / 2)) / scale;
float y1 = (faceobjects[i].rect.y + faceobjects[i].rect.height - (hpad / 2)) / scale;
x0 = std::max(std::min(x0, (float)width - 1), 0.f);
y0 = std::max(std::min(y0, (float)height - 1), 0.f);
x1 = std::max(std::min(x1, (float)width - 1), 0.f);
y1 = std::max(std::min(y1, (float)height - 1), 0.f);
faceobjects[i].rect.x = x0;
faceobjects[i].rect.y = y0;
faceobjects[i].rect.width = x1 - x0;
faceobjects[i].rect.height = y1 - y0;
if (has_kps)
{
float x0 = (faceobjects[i].landmark[0].x - (wpad / 2)) / scale;
float y0 = (faceobjects[i].landmark[0].y - (hpad / 2)) / scale;
float x1 = (faceobjects[i].landmark[1].x - (wpad / 2)) / scale;
float y1 = (faceobjects[i].landmark[1].y - (hpad / 2)) / scale;
float x2 = (faceobjects[i].landmark[2].x - (wpad / 2)) / scale;
float y2 = (faceobjects[i].landmark[2].y - (hpad / 2)) / scale;
float x3 = (faceobjects[i].landmark[3].x - (wpad / 2)) / scale;
float y3 = (faceobjects[i].landmark[3].y - (hpad / 2)) / scale;
float x4 = (faceobjects[i].landmark[4].x - (wpad / 2)) / scale;
float y4 = (faceobjects[i].landmark[4].y - (hpad / 2)) / scale;
faceobjects[i].landmark[0].x = std::max(std::min(x0, (float)width - 1), 0.f);
faceobjects[i].landmark[0].y = std::max(std::min(y0, (float)height - 1), 0.f);
faceobjects[i].landmark[1].x = std::max(std::min(x1, (float)width - 1), 0.f);
faceobjects[i].landmark[1].y = std::max(std::min(y1, (float)height - 1), 0.f);
faceobjects[i].landmark[2].x = std::max(std::min(x2, (float)width - 1), 0.f);
faceobjects[i].landmark[2].y = std::max(std::min(y2, (float)height - 1), 0.f);
faceobjects[i].landmark[3].x = std::max(std::min(x3, (float)width - 1), 0.f);
faceobjects[i].landmark[3].y = std::max(std::min(y3, (float)height - 1), 0.f);
faceobjects[i].landmark[4].x = std::max(std::min(x4, (float)width - 1), 0.f);
faceobjects[i].landmark[4].y = std::max(std::min(y4, (float)height - 1), 0.f);
}
}
return 0;
}
int SCRFD::readModels(std::string param_path, std::string model_path, bool use_gpu)
{
bool has_gpu = false;
#if NCNN_VULKAN
ncnn::create_gpu_instance();
has_gpu = ncnn::get_gpu_count() > 0;
#endif
bool to_use_gpu = has_gpu && use_gpu;
scrfd_net.opt.use_vulkan_compute = to_use_gpu;
int rp = scrfd_net.load_param(param_path.c_str());
int rb = scrfd_net.load_model(model_path.c_str());
if (rp < 0 || rb < 0)
{
return 1;
}
return 0;
}
2.把檢測的結果畫出來，
int SCRFD::draw(cv::Mat& rgb, const std::vector<FaceObject>& faceobjects)
{
for (size_t i = 0; i < faceobjects.size(); i++)
{
const FaceObject& obj = faceobjects[i];
cv::rectangle(rgb, obj.rect, cv::Scalar(0, 255, 0));
if (has_kps)
{
cv::circle(rgb, obj.landmark[0], 2, cv::Scalar(0, 255, 255), -1);
cv::circle(rgb, obj.landmark[1], 2, cv::Scalar(0, 0, 255), -1);
cv::circle(rgb, obj.landmark[2], 2, cv::Scalar(255, 255, 0), -1);
cv::circle(rgb, obj.landmark[3], 2, cv::Scalar(255, 255, 0), -1);
cv::circle(rgb, obj.landmark[4], 2, cv::Scalar(255, 255, 0), -1);
}
char text[256];
sprintf(text, "%.1f%%", obj.prob * 100);
int baseLine = 0;
cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine);
int x = obj.rect.x;
int y = obj.rect.y - label_size.height - baseLine;
if (y < 0)
y = 0;
if (x + label_size.width > rgb.cols)
x = rgb.cols - label_size.width;
cv::rectangle(rgb, cv::Rect(cv::Point(x, y), cv::Size(label_size.width, label_size.height + baseLine)), cv::Scalar(255, 255, 255), -1);
cv::putText(rgb, text, cv::Point(x, y + label_size.height), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 0, 0), 1);
}
return 0;
}
3.檢測效果
三.證件照剪切
1.篩選人臉，如果有一張影像有多張人臉的話，取最大最正的臉的坐標來做基準點，
代碼：
int faceFind(const cv::Mat& cv_src, std::vector<FaceObject> &face_object, cv::Rect& cv_rect, std::vector<cv::Point> &five_point)
{
//只檢測到一張臉
if (face_object.size() == 1)
{
if (face_object[0].prob > 0.7)
{
for (int i = 0; i < 5; ++i)
{
five_point.push_back(face_object[0].landmark[i]);
}
cv_rect = face_object[0].rect;
return 0;
}
}
//檢測到多張臉
else if (face_object.size() >= 2)
{
cv::Rect max_rect;
for (int i = 0; i < face_object.size(); ++i)
{
if (face_object[i].prob >= 0.7)
{
cv::Rect rect = face_object[i].rect;
if (max_rect.area() <= rect.area())
{
max_rect = rect;
}
}
}
for (int i = 0; i < face_object.size(); ++i)
{
if (face_object[i].prob >= 0.7)
{
cv::Rect rect = face_object[i].rect;
if (max_rect.area() == rect.area())
{
for (int j = 0; j < 5; ++j)
{
five_point.push_back(face_object[0].landmark[j]);
}
cv_rect = rect;
}
}
}
return 0;
}
return 1;
}
效果：
2.上面取基準的方法只是一個比較簡單的方法，如果算力夠的話，或者需要精度更高的話，這里可以加入更多關鍵點和頭部姿態估計和判斷，然后用頭部姿態估計來判斷影像或者攝像頭頭里的人臉是否擺正了，
3.以人臉為基準剪切出證件照的尺寸影像，先把臉基準中心，計算上下左右的尺寸，然后按比例剪切出合適的證件照的尺寸，
代碼：
int faceLocation(const cv::Mat cv_src,cv::Mat& cv_dst, std::vector<cv::Point>& five_point, cv::Rect &cv_rect)
{
float w_block = cv_rect.width / 5.5;
float h_block = cv_rect.height / 8;
//頭部
cv::Rect face_rect;
face_rect.x = cv_rect.x - (w_block * 0.8);//加上雙耳的大小
face_rect.y = cv_rect.y - (h_block * 2);
face_rect.width = cv_rect.width + (w_block * 1.6);
face_rect.height = cv_rect.height + (h_block * 2);
//人臉離左邊邊框的距離
int tl_face_w = face_rect.tl().x;
int tr_face_w = cv_src.cols - (face_rect.width + face_rect.tl().x);
int t_face_h = face_rect.tl().y;
int b_face_h = cv_src.rows - face_rect.br().y;
//算出頭像的位置
int w_scale = face_rect.width / 7;
int h_scale = face_rect.height / 10;
cv::Rect id_rect;
//判斷位置
if (tl_face_w >= (w_scale * 2) && tr_face_w >= (w_scale * 2) && t_face_h >= (h_scale * 0.5) && b_face_h > (h_scale * 5))
{
//判斷眼睛的位置
std::cout << five_point.size() << std::endl;
if (abs(five_point.at(0).y - five_point.at(1).y) < 8)
{
id_rect.x = ((face_rect.x - w_scale * 3) <= 0) ? 0 : (face_rect.x - w_scale * 3);
id_rect.y = ((face_rect.y - h_scale * 3) < 0) ? 0 : (face_rect.y - h_scale * 3);
id_rect.width = (w_scale * 13) + id_rect.x > cv_src.size().width ? cv_src.size().width - id_rect.x : w_scale * 13;
id_rect.height = (h_scale * 19) + id_rect.y > cv_src.size().height ? cv_src.size().height - id_rect.y : h_scale * 19;
cv_dst = cv_src(id_rect);
return 0;
}
}
return -1;
}
效果：
四.摳圖與背景替換
1.經過上面的步驟，已經得到一個證件照的影像，現在要把頭像摳出來就可以做背景替換了，
int matting(cv::Mat &cv_src, ncnn::Net& net, ncnn::Mat &alpha)
{
int width = cv_src.cols;
int height = cv_src.rows;
ncnn::Mat in_resize = ncnn::Mat::from_pixels_resize(cv_src.data, ncnn::Mat::PIXEL_RGB, width, height, 256,256);
const float meanVals[3] = { 127.5f, 127.5f, 127.5f };
const float normVals[3] = { 0.0078431f, 0.0078431f, 0.0078431f };
in_resize.substract_mean_normalize(meanVals, normVals);
ncnn::Mat out;
ncnn::Extractor ex = net.create_extractor();
ex.set_vulkan_compute(true);
ex.input("input", in_resize);
ex.extract("output", out);
ncnn::resize_bilinear(out, alpha, width, height);
return 0;
}
2.替換背景色，
void replaceBG(const cv::Mat cv_src, ncnn::Mat &alpha,cv::Mat &cv_matting, std::vector<int> &bg_color)
{
int width = cv_src.cols;
int height = cv_src.rows;
cv_matting = cv::Mat::zeros(cv::Size(width, height), CV_8UC3);
float* alpha_data = (float*)alpha.data;
for (int i = 0; i < height; i++)
{
for (int j = 0; j < width; j++)
{
float alpha_ = alpha_data[i * width + j];
cv_matting.at < cv::Vec3b>(i, j)[0] = cv_src.at < cv::Vec3b>(i, j)[0] * alpha_ + (1 - alpha_) * bg_color[0];
cv_matting.at < cv::Vec3b>(i, j)[1] = cv_src.at < cv::Vec3b>(i, j)[1] * alpha_ + (1 - alpha_) * bg_color[1];
cv_matting.at < cv::Vec3b>(i, j)[2] = cv_src.at < cv::Vec3b>(i, j)[2] * alpha_ + (1 - alpha_) * bg_color[2];
}
}
}
3.效果圖，
原圖：
證件照：
原圖（背景比較復雜的原圖）：
證件照：
動漫頭像：
五.結語
1.這只是個可以實作功能的demo，如果想要應用到商業上，還有很多細節上的處理，比如果頭部姿態估計，眼球檢測（是否閉眼），皮膚美化，瘦臉，換裝等，這些功能有時間我會去試之后放上來，
2.這個demo改改可以在安卓上運行，demo我在安卓上測驗過，速度和精度都有不錯的表現，
3.整個工程和原始碼的地址：https://download.csdn.net/download/matt45m/67756246
轉載請註明出處，本文鏈接：https://www.uj5u.com/yidong/389198.html
標籤：其他