opencv獲取視頻影像在web端無插件播放解決方案
- 一、步驟說明
- 1、opencv獲取影像Mat
- 2、Mat實時轉H264
- 3、服務端使用Qt通過websocket發送
- 4、前端使用wfs.js進行接收，播放
- 二、具體分析
一、步驟說明
1、opencv獲取影像Mat
2、Mat實時轉H264
3、服務端使用Qt通過websocket發送
4、前端使用wfs.js進行接收，播放
綜合分析以及心得：
由于接到的專案中需要對視頻中某一區域進行截取并進行相關影像演算法處理（此處不進行介紹），并且Web前端進行無插件顯示播放，所以這里使用opencv進行視頻影像的獲取，之前的做法是通過ffmpeg將獲取的視頻幀進行推流到nginx-rtmp服務器中，前端通過http訪問m3u8進行實時讀取，最后發現該方法延遲性極高（大概10s左右），放棄了該方法，最后想到通過websocket進行前端直接通信，這樣可以降低延遲，所以在github中發現了wfs.js該方法，但是這里面需要傳入H264編碼的裸流，所以經過周折終于把mat轉為了H264，(并且影像大小可以任意，專案需要)，
二、具體分析
1、opencv獲取影像Mat
opencv獲取視頻使用VideoCapture直接進行獲取，具體代碼如下
cv::VideoCapture cam;
cam.open(inUrl); //輸入源
//cam.open(0); // 本地相機
if (!cam.isOpened())
{
throw CException("cam open failed!");
}
fps = cam.get(CAP_PROP_FPS);
fps = 25;
Mat frame;
for (;;)
{
if(isNeedStop)
{
break;
}
///讀取rtsp視頻幀，解碼視頻幀
if (!cam.grab())
{
continue;
}
///yuv轉換為rgb
if (!cam.retrieve(frame))
{
continue;
}
QThread::msleep(1000/fps);
}
2、Mat實時轉H264
Mat轉H264裸流這里我們直接ffmpeg進行轉碼，廢話不多說直接上代碼
h264encoder.h
#ifndef H264ENCODER_H
#define H264ENCODER_H
#include <QObject>
#include "pch.h"
#include "opencv2/opencv.hpp"
typedef struct AvH264EncConfig_T {
int width = 320;
int height = 240;
int frame_rate = 25;
// int64_t bit_rate = 320000;
int64_t bit_rate = 50 * 1024 * 8;
int gop_size = 50;
int max_b_frames = 0;
}AvH264EncConfig;
class h264encoder : public QObject
{
Q_OBJECT
public:
explicit h264encoder(QObject *parent = nullptr);
~h264encoder();
int Init(AvH264EncConfig h264_config);
AVPacket *encode(const cv::Mat& mat);
void Destory();
private:
AVFrame* cvmatToAvframe(const cv::Mat* image, AVFrame* frame);
AVCodec *cdc_;
AVCodecContext *cdc_ctx_;
AVFrame *avf_;
AVPacket *avp_;
int frame_size_;
int pts_;
signals:
void sig_GetOneFrame(QByteArray);
public slots:
void slot_encode(const cv::Mat& mat);
};
#endif // H264ENCODER_H
h264encoder.cpp
#include "h264encoder.h"
#include <QDebug>
h264encoder::h264encoder(QObject *parent) : QObject(parent)
{
cdc_ = NULL;
cdc_ctx_ = NULL;
avf_ = NULL;
avp_ = NULL;
}
h264encoder::~h264encoder()
{
Destory();
}
int h264encoder::Init(AvH264EncConfig h264_config) {
qDebug()<<"h264 init";
pts_ = 0;
cdc_ = avcodec_find_encoder(AV_CODEC_ID_H264);
if (!cdc_) {
return -1;
}
cdc_ctx_ = avcodec_alloc_context3(cdc_);
if (!cdc_ctx_) {
return -1;
}
cdc_ctx_->bit_rate = h264_config.bit_rate;//導致畫面模糊
cdc_ctx_->width = h264_config.width;
cdc_ctx_->height = h264_config.height;
cdc_ctx_->time_base = { 1, h264_config.frame_rate };
cdc_ctx_->framerate = { h264_config.frame_rate, 1 };
cdc_ctx_->gop_size = h264_config.gop_size;
cdc_ctx_->max_b_frames = h264_config.max_b_frames;
cdc_ctx_->pix_fmt = AV_PIX_FMT_YUV420P;
cdc_ctx_->codec_id = AV_CODEC_ID_H264;
cdc_ctx_->codec_type = AVMEDIA_TYPE_VIDEO;
//cdc_ctx_->qmin = 10;
//cdc_ctx_->qmax = 51;
//cdc_ctx_->qcompress = 0.6;
AVDictionary *dict = nullptr;
//av_dict_set(&dict, "preset", "slow", 0);
av_dict_set(&dict, "preset", "veryfast", 0);
av_dict_set(&dict, "tune", "zerolatency", 0);
av_dict_set(&dict, "profile", "main", 0);
// av_dict_set(&dict, "bufsize", "131072", 0);
avf_ = av_frame_alloc();
avp_ = av_packet_alloc();
if (!avf_ || !avp_) {
return -1;
}
av_init_packet(avp_);
frame_size_ = cdc_ctx_->width * cdc_ctx_->height;
avf_->format = cdc_ctx_->pix_fmt;
avf_->width = cdc_ctx_->width;
avf_->height = cdc_ctx_->height;
// alloc memory
int r = av_frame_get_buffer(avf_, 0);
if (r < 0) {
return -1;
}
r = av_frame_make_writable(avf_);
if (r < 0) {
return -1;
}
return avcodec_open2(cdc_ctx_, cdc_, &dict);
}
void h264encoder::Destory() {
if(cdc_ctx_) avcodec_free_context(&cdc_ctx_);
if (avf_) av_frame_free(&avf_);
if (avp_) av_packet_free(&avp_);
}
AVPacket *h264encoder::encode(const cv::Mat& mat) {
if (mat.empty()) return NULL;
avf_ = cvmatToAvframe(&mat,avf_);
int r = avcodec_send_frame(cdc_ctx_, avf_);
if (r >= 0) {
r = avcodec_receive_packet(cdc_ctx_, avp_);
if (r == 0) {
avp_->stream_index = avf_->pts;
return avp_;
}
if (r == AVERROR(EAGAIN) || r == AVERROR_EOF) {
return NULL;
}
}
return NULL;
}
AVFrame* h264encoder::cvmatToAvframe(const cv::Mat* image, AVFrame* frame){
int width = image->cols;
int height = image->rows;
int cvLinesizes[1];
cvLinesizes[0] = image->step1();
if (frame == NULL){
frame = av_frame_alloc();
av_image_alloc(frame->data, frame->linesize, width, height, AVPixelFormat::AV_PIX_FMT_YUV420P, 1);
}
SwsContext* conversion = sws_getContext(width, height, AVPixelFormat::AV_PIX_FMT_BGR24, width, height, (AVPixelFormat) frame->format, SWS_FAST_BILINEAR, NULL, NULL, NULL);
sws_scale(conversion, &image->data, cvLinesizes , 0, height, frame->data, frame->linesize);
sws_freeContext(conversion);
return frame;
}
void h264encoder::slot_encode(const cv::Mat& mat)
{
AVPacket *packet = encode(mat);
QByteArray data;
data.append((char*)packet->data, packet->size);
emit sig_GetOneFrame(data);
}
#include "h264encoder.h"
#include <QDebug>
h264encoder::h264encoder(QObject *parent) : QObject(parent)
{
cdc_ = NULL;
cdc_ctx_ = NULL;
avf_ = NULL;
avp_ = NULL;
}
h264encoder::~h264encoder()
{
Destory();
}
int h264encoder::Init(AvH264EncConfig h264_config) {
qDebug()<<"h264 init";
pts_ = 0;
cdc_ = avcodec_find_encoder(AV_CODEC_ID_H264);
if (!cdc_) {
return -1;
}
cdc_ctx_ = avcodec_alloc_context3(cdc_);
if (!cdc_ctx_) {
return -1;
}
cdc_ctx_->bit_rate = h264_config.bit_rate;//導致畫面模糊
cdc_ctx_->width = h264_config.width;
cdc_ctx_->height = h264_config.height;
cdc_ctx_->time_base = { 1, h264_config.frame_rate };
cdc_ctx_->framerate = { h264_config.frame_rate, 1 };
cdc_ctx_->gop_size = h264_config.gop_size;
cdc_ctx_->max_b_frames = h264_config.max_b_frames;
cdc_ctx_->pix_fmt = AV_PIX_FMT_YUV420P;
cdc_ctx_->codec_id = AV_CODEC_ID_H264;
cdc_ctx_->codec_type = AVMEDIA_TYPE_VIDEO;
//cdc_ctx_->qmin = 10;
//cdc_ctx_->qmax = 51;
//cdc_ctx_->qcompress = 0.6;
AVDictionary *dict = nullptr;
//av_dict_set(&dict, "preset", "slow", 0);
av_dict_set(&dict, "preset", "veryfast", 0);
av_dict_set(&dict, "tune", "zerolatency", 0);
av_dict_set(&dict, "profile", "main", 0);
// av_dict_set(&dict, "bufsize", "131072", 0);
avf_ = av_frame_alloc();
avp_ = av_packet_alloc();
if (!avf_ || !avp_) {
return -1;
}
av_init_packet(avp_);
frame_size_ = cdc_ctx_->width * cdc_ctx_->height;
avf_->format = cdc_ctx_->pix_fmt;
avf_->width = cdc_ctx_->width;
avf_->height = cdc_ctx_->height;
// alloc memory
int r = av_frame_get_buffer(avf_, 0);
if (r < 0) {
return -1;
}
r = av_frame_make_writable(avf_);
if (r < 0) {
return -1;
}
return avcodec_open2(cdc_ctx_, cdc_, &dict);
}
void h264encoder::Destory() {
if(cdc_ctx_) avcodec_free_context(&cdc_ctx_);
if (avf_) av_frame_free(&avf_);
if (avp_) av_packet_free(&avp_);
}
AVPacket *h264encoder::encode(const cv::Mat& mat) {
if (mat.empty()) return NULL;
avf_ = cvmatToAvframe(&mat,avf_);
int r = avcodec_send_frame(cdc_ctx_, avf_);
if (r >= 0) {
r = avcodec_receive_packet(cdc_ctx_, avp_);
if (r == 0) {
avp_->stream_index = avf_->pts;
return avp_;
}
if (r == AVERROR(EAGAIN) || r == AVERROR_EOF) {
return NULL;
}
}
return NULL;
}
AVFrame* h264encoder::cvmatToAvframe(const cv::Mat* image, AVFrame* frame){
int width = image->cols;
int height = image->rows;
int cvLinesizes[1];
cvLinesizes[0] = image->step1();
if (frame == NULL){
frame = av_frame_alloc();
av_image_alloc(frame->data, frame->linesize, width, height, AVPixelFormat::AV_PIX_FMT_YUV420P, 1);
}
SwsContext* conversion = sws_getContext(width, height, AVPixelFormat::AV_PIX_FMT_BGR24, width, height, (AVPixelFormat) frame->format, SWS_FAST_BILINEAR, NULL, NULL, NULL);
sws_scale(conversion, &image->data, cvLinesizes , 0, height, frame->data, frame->linesize);
sws_freeContext(conversion);
return frame;
}
void h264encoder::slot_encode(const cv::Mat& mat)
{
AVPacket *packet = encode(mat);
QByteArray data;
data.append((char*)packet->data, packet->size);
emit sig_GetOneFrame(data);
}
使用解釋：這里使用Qt的信號槽機制，通過將opencv產生的Mat傳入到槽函式slot_encode，然后進行解碼，首先將Mat轉碼為AvFrame幀，之后編碼為H264格式的視頻幀，之后通過信號將資料轉為而二進制發送出去
3、服務端使用Qt通過websocket發送
Qt建立websocket進行發送webSocket->sendBinaryMessage(data);
4、前端使用wfs.js進行接收，播放
wfs具體內容見github 地址：https://github.com/ChihChengYang/wfs.js
此處給出測驗使用方法：
index.html
<html>
<head>
<title>RTSP PLAY WITH MSE</title>
<style>
.video_bg {
background-image: url("video.png");
background-repeat: no-repeat;
background-size: 100% 100%;
}
.video_size {
height: 450px;
width: 800px;
}
#rtsp_url {
width: 400px;
}
/*video默認全屏按鈕*/
video::-webkit-media-controls-fullscreen-button {
display: none !important;
}
/*video默認aduio音量按鈕*/
video::-webkit-media-controls-mute-button {
display: none !important;
}
/*video默認setting按鈕*/
video::-internal-media-controls-overflow-button {
display: none !important;
}
/*禁用video的controls（要慎重！不要輕易隱藏掉，會導致點擊視頻不能播放）*/
video::-webkit-media-controls {
display: none !important;
}
</style>
</head>
<body>
<script type="text/javascript" src="jquery-1.12.1.min.js"></script>
<script type="text/javascript" src="wfs.js"></script>
<div class="wfsjs">
<video id="video1" class="video_size" autoplay="autoplay" loop="false" poster="./video.png"></video>
<div class="inputDiv">
<input id="ws_url" type="text" placeholder="請輸入WebSocket Url" value="ws://192.168.1.193:48780" />
<input id="rtsp_url" type="text" placeholder="請輸入RTSP URL..." value="rtsp://admin:long7854016@192.168.1.20:554/h264/ch1/main/av_stream" />
<button id="play" class="plays" onclick="play()">播 放</button>
<button id="stop" onclick="stop()">停 止</button>
<button id="fullScreenBtn">全屏</button>
</div>
</div>
<script>
var G_isHiden = false;
var wfs = null;
window.onload = function() {
if (!Wfs.isSupported()) {
alert("您的瀏覽器不支持 MediaSource Extend,某些功能使用將受到影響！")
}
};
function play() {
if (wfs) {
stop();
}
wfs = new Wfs();
var ws_url = $("#ws_url").val();
var rtsp_url = $("#rtsp_url").val();
if (ws_url == "") {
alert("請填寫WebSocket Url");
return false;
}
if (rtsp_url == "") {
alert("請填寫RTSP Url");
return false;
}
var video1 = document.getElementsByClassName("video_size")[0];
console.log("rtsp:"+rtsp_url);
wfs.attachMedia(video1, rtsp_url, "H264Raw", ws_url);
}
function stop() {
wfs.closeWebSocket();
wfs.destroy();
}
document.getElementById('fullScreenBtn').addEventListener('click', function() {
FullScreen('video1');
})
//進入全屏
function FullScreen(id) {
var ele = document.getElementById(id);
if (ele.requestFullscreen) {
ele.requestFullscreen();
} else if (ele.mozRequestFullScreen) {
ele.mozRequestFullScreen();
} else if (ele.webkitRequestFullScreen) {
ele.webkitRequestFullScreen();
} else if (ele.msRequestFullscreen) {
ele.msRequestFullscreen();
} else if (ele.oRequestFullscreen) {
ele.oRequestFullscreen();
}
}
//退出全屏
function exitFullscreen() {
var de = document;
if (de.exitFullscreen) {
de.exitFullscreen();
} else if (de.mozCancelFullScreen) {
de.mozCancelFullScreen();
} else if (de.webkitExitFullscreen) {
de.webkitExitFullscreen();
} else if (de.msExitFullscreen) {
de.msExitFullscreen();
} else if (de.oRequestFullscreen) {
de.oCancelFullScreen();
}
}
document.addEventListener('visibilitychange', function() {
var isHidden = document.hidden;
if (isHidden) {
Wfs.DefaultConfig.isHidden = true;
} else {
Wfs.DefaultConfig.isHidden = false;
}
});
</script>
</body>
</html>
轉載請註明出處，本文鏈接：https://www.uj5u.com/qita/261823.html
標籤：其他