Go语言视频流rtsp转webrtc
xiaoyaoyou.xyz 人气:01. 前言
前面我们测试了rtsp转hls方式，发现延迟比较大，不太适合我们的使用需求。接下来我们试一下webrtc的方式看下使用情况。
综合考虑下来，我们最好能找到一个go作为后端，前端兼容性较好的前后端方案来处理webrtc，这样我们就可以结合我们之前的cgo+onvif+gSoap实现方案来获取rtsp流，并且可以根据已经实现的ptz、预置点等功能接口做更多的扩展。
2. rtsp转webRTC
如下是找到的一个比较合适的开源方案，前端使用了jQuery、bootstrap等，后端使用go+gin来实现并将rtsp流解析转换为webRTC协议提供http相关接口给到前端，通过config.json配置rtsp地址和stun地址：
此外，还带有stun，可以自行配置stun地址，便于进行内网穿透。
初步测试几乎看不出来延迟，符合预期，使用的jQuery+bootstrap+go+gin做的web，也符合我们的项目使用情况。
3. 初步测试结果
结果如下：
4. 结合我们之前的onvif+gSoap+cgo的方案做修改
我们在此项目的基础上，结合我们之前研究的onvif+cgo+gSoap的方案，将onvif获取到的相关数据提供接口到web端，增加ptz、调焦、缩放等功能。
我们在http.go中添加新的post接口：HTTPAPIServerStreamPtz来进行ptz和HTTPAPIServerStreamPreset进行预置点相关操作。
以下是部分代码，没有做太多的优化，也仅仅实现了ptz、调焦和缩放，算是打通了通路，具体项目需要可以再做优化。
4.1 go后端修改
增加了新的接口，并将之前cgo+onvif+gSoap的内容结合了进来，项目整体没有做更多的优化，只是为了演示，提供一个思路：
http.go（增加了两个post接口ptz和preset，采用cgo方式处理）：
package main
/*
#cgo CFLAGS: -I ./ -I /usr/local/
#cgo LDFLAGS: -L ./build -lc_onvif_static -lpthread -ldl -lssl -lcrypto
#include "client.h"
#include "malloc.h"
*/
import "C"
import (
"encoding/json"
"fmt"
"log"
"net/http"
"os"
"sort"
"strconv"
"time"
"unsafe"
"github.com/deepch/vdk/av"
webrtc "github.com/deepch/vdk/format/webrtcv3"
"github.com/gin-gonic/gin"
)
type JCodec struct {
Type string
}
func serveHTTP() {
gin.SetMode(gin.ReleaseMode)
router := gin.Default()
router.Use(CORSMiddleware())
if _, err := os.Stat("./web"); !os.IsNotExist(err) {
router.LoadHTMLGlob("web/templates/*")
router.GET("/", HTTPAPIServerIndex)
router.GET("/stream/player/:uuid", HTTPAPIServerStreamPlayer)
}
router.POST("/stream/receiver/:uuid", HTTPAPIServerStreamWebRTC)
//增加新的post接口
router.POST("/stream/ptz/", HTTPAPIServerStreamPtz)
router.POST("/stream/preset/", HTTPAPIServerStreamPreset)
router.GET("/stream/codec/:uuid", HTTPAPIServerStreamCodec)
router.POST("/stream", HTTPAPIServerStreamWebRTC2)
router.StaticFS("/static", http.Dir("web/static"))
err := router.Run(Config.Server.HTTPPort)
if err != nil {
log.Fatalln("Start HTTP Server error", err)
}
}
//HTTPAPIServerIndex index
func HTTPAPIServerIndex(c *gin.Context) {
_, all := Config.list()
if len(all) > 0 {
c.Header("Cache-Control", "no-cache, max-age=0, must-revalidate, no-store")
c.Header("Access-Control-Allow-Origin", "*")
c.Redirect(http.StatusMovedPermanently, "stream/player/"+all[0])
} else {
c.HTML(http.StatusOK, "index.tmpl", gin.H{
"port": Config.Server.HTTPPort,
"version": time.Now().String(),
})
}
}
//HTTPAPIServerStreamPlayer stream player
func HTTPAPIServerStreamPlayer(c *gin.Context) {
_, all := Config.list()
sort.Strings(all)
c.HTML(http.StatusOK, "player.tmpl", gin.H{
"port": Config.Server.HTTPPort,
"suuid": c.Param("uuid"),
"suuidMap": all,
"version": time.Now().String(),
})
}
//HTTPAPIServerStreamCodec stream codec
func HTTPAPIServerStreamCodec(c *gin.Context) {
if Config.ext(c.Param("uuid")) {
Config.RunIFNotRun(c.Param("uuid"))
codecs := Config.coGe(c.Param("uuid"))
if codecs == nil {
return
}
var tmpCodec []JCodec
for _, codec := range codecs {
if codec.Type() != av.H264 && codec.Type() != av.PCM_ALAW && codec.Type() != av.PCM_MULAW && codec.Type() != av.OPUS {
log.Println("Codec Not Supported WebRTC ignore this track", codec.Type())
continue
}
if codec.Type().IsVideo() {
tmpCodec = append(tmpCodec, JCodec{Type: "video"})
} else {
tmpCodec = append(tmpCodec, JCodec{Type: "audio"})
}
}
b, err := json.Marshal(tmpCodec)
if err == nil {
_, err = c.Writer.Write(b)
if err != nil {
log.Println("Write Codec Info error", err)
return
}
}
}
}
//HTTPAPIServerStreamWebRTC stream video over WebRTC
func HTTPAPIServerStreamWebRTC(c *gin.Context) {
if !Config.ext(c.PostForm("suuid")) {
log.Println("Stream Not Found")
return
}
Config.RunIFNotRun(c.PostForm("suuid"))
codecs := Config.coGe(c.PostForm("suuid"))
if codecs == nil {
log.Println("Stream Codec Not Found")
return
}
var AudioOnly bool
if len(codecs) == 1 && codecs[0].Type().IsAudio() {
AudioOnly = true
}
muxerWebRTC := webrtc.NewMuxer(webrtc.Options{ICEServers: Config.GetICEServers(), ICEUsername: Config.GetICEUsername(), ICECredential: Config.GetICECredential(), PortMin: Config.GetWebRTCPortMin(), PortMax: Config.GetWebRTCPortMax()})
answer, err := muxerWebRTC.WriteHeader(codecs, c.PostForm("data"))
if err != nil {
log.Println("WriteHeader", err)
return
}
_, err = c.Writer.Write([]byte(answer))
if err != nil {
log.Println("Write", err)
return
}
go func() {
cid, ch := Config.clAd(c.PostForm("suuid"))
defer Config.clDe(c.PostForm("suuid"), cid)
defer muxerWebRTC.Close()
var videoStart bool
noVideo := time.NewTimer(10 * time.Second)
for {
select {
case <-noVideo.C:
log.Println("noVideo")
return
case pck := <-ch:
if pck.IsKeyFrame || AudioOnly {
noVideo.Reset(10 * time.Second)
videoStart = true
}
if !videoStart && !AudioOnly {
continue
}
err = muxerWebRTC.WritePacket(pck)
if err != nil {
log.Println("WritePacket", err)
return
}
}
}
}()
}
func HTTPAPIServerStreamPtz(c *gin.Context) {
action := c.PostForm("action")
direction, err := strconv.Atoi(action)
if err != nil {
log.Println(err)
return
}
var soap C.P_Soap
soap = C.new_soap(soap)
username := C.CString("admin")
password := C.CString("admin")
serviceAddr := C.CString("http://40.40.40.101:80/onvif/device_service")
C.get_device_info(soap, username, password, serviceAddr)
mediaAddr := [200]C.char{}
C.get_capabilities(soap, username, password, serviceAddr, &mediaAddr[0])
profileToken := [200]C.char{}
C.get_profiles(soap, username, password, &profileToken[0], &mediaAddr[0])
videoSourceToken := [200]C.char{}
C.get_video_source(soap, username, password, &videoSourceToken[0], &mediaAddr[0])
switch direction {
case 0:
break
case 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11:
C.ptz(soap, username, password, C.int(direction), C.float(0.5), &profileToken[0], &mediaAddr[0])
case 12, 13, 14:
C.focus(soap, username, password, C.int(direction), C.float(0.5), &videoSourceToken[0], &mediaAddr[0])
default:
fmt.Println("Unknown direction.")
}
C.del_soap(soap)
C.free(unsafe.Pointer(username))
C.free(unsafe.Pointer(password))
C.free(unsafe.Pointer(serviceAddr))
c.JSON(http.StatusOK, gin.H{"message":"success"})
}
func HTTPAPIServerStreamPreset(c *gin.Context) {
var soap C.P_Soap
soap = C.new_soap(soap)
username := C.CString("admin")
password := C.CString("admin")
serviceAddr := C.CString("http://40.40.40.101:80/onvif/device_service")
C.get_device_info(soap, username, password, serviceAddr)
mediaAddr := [200]C.char{}
C.get_capabilities(soap, username, password, serviceAddr, &mediaAddr[0])
profileToken := [200]C.char{}
C.get_profiles(soap, username, password, &profileToken[0], &mediaAddr[0])
videoSourceToken := [200]C.char{}
C.get_video_source(soap, username, password, &videoSourceToken[0], &mediaAddr[0])
action := c.PostForm("action")
presetAction, err := strconv.Atoi(action)
if err != nil {
log.Println(err)
return
}
fmt.Println("请输入数字进行preset,1-4分别代表查询、设置、跳转、删除预置点；退出输入0:")
switch presetAction {
case 0:
break
case 1:
C.preset(soap, username, password, C.int(presetAction), nil, nil, &profileToken[0], &mediaAddr[0])
case 2:
fmt.Println("请输入要设置的预置点token信息：")
presentToken := ""
_, _ = fmt.Scanln(&presentToken)
fmt.Println("请输入要设置的预置点name信息长度不超过200：")
presentName := ""
_, _ = fmt.Scanln(&presentName)
C.preset(soap, username, password, C.int(presetAction), C.CString(presentToken), C.CString(presentName), &profileToken[0], &mediaAddr[0])
case 3:
fmt.Println("请输入要跳转的预置点token信息：")
presentToken := ""
_, _ = fmt.Scanln(&presentToken)
C.preset(soap, username, password, C.int(presetAction), C.CString(presentToken), nil, &profileToken[0], &mediaAddr[0])
case 4:
fmt.Println("请输入要删除的预置点token信息：")
presentToken := ""
_, _ = fmt.Scanln(&presentToken)
C.preset(soap, username, password, C.int(presetAction), C.CString(presentToken), nil, &profileToken[0], &mediaAddr[0])
default:
fmt.Println("unknown present action.")
break
}
C.del_soap(soap)
C.free(unsafe.Pointer(username))
C.free(unsafe.Pointer(password))
C.free(unsafe.Pointer(serviceAddr))
c.JSON(http.StatusOK, gin.H{"message":"success"})
}
func CORSMiddleware() gin.HandlerFunc {
return func(c *gin.Context) {
c.Header("Access-Control-Allow-Origin", "*")
c.Header("Access-Control-Allow-Credentials", "true")
c.Header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept, Authorization, x-access-token")
c.Header("Access-Control-Expose-Headers", "Content-Length, Access-Control-Allow-Origin, Access-Control-Allow-Headers, Cache-Control, Content-Language, Content-Type")
c.Header("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT")
if c.Request.Method == "OPTIONS" {
c.AbortWithStatus(http.StatusNoContent)
return
}
c.Next()
}
}
type Response struct {
Tracks []string `json:"tracks"`
Sdp64 string `json:"sdp64"`
}
type ResponseError struct {
Error string `json:"error"`
}
func HTTPAPIServerStreamWebRTC2(c *gin.Context) {
url := c.PostForm("url")
if _, ok := Config.Streams[url]; !ok {
Config.Streams[url] = StreamST{
URL: url,
OnDemand: true,
Cl: make(map[string]viewer),
}
}
Config.RunIFNotRun(url)
codecs := Config.coGe(url)
if codecs == nil {
log.Println("Stream Codec Not Found")
c.JSON(500, ResponseError{Error: Config.LastError.Error()})
return
}
muxerWebRTC := webrtc.NewMuxer(
webrtc.Options{
ICEServers: Config.GetICEServers(),
PortMin: Config.GetWebRTCPortMin(),
PortMax: Config.GetWebRTCPortMax(),
},
)
sdp64 := c.PostForm("sdp64")
answer, err := muxerWebRTC.WriteHeader(codecs, sdp64)
if err != nil {
log.Println("Muxer WriteHeader", err)
c.JSON(500, ResponseError{Error: err.Error()})
return
}
response := Response{
Sdp64: answer,
}
for _, codec := range codecs {
if codec.Type() != av.H264 &&
codec.Type() != av.PCM_ALAW &&
codec.Type() != av.PCM_MULAW &&
codec.Type() != av.OPUS {
log.Println("Codec Not Supported WebRTC ignore this track", codec.Type())
continue
}
if codec.Type().IsVideo() {
response.Tracks = append(response.Tracks, "video")
} else {
response.Tracks = append(response.Tracks, "audio")
}
}
c.JSON(200, response)
AudioOnly := len(codecs) == 1 && codecs[0].Type().IsAudio()
go func() {
cid, ch := Config.clAd(url)
defer Config.clDe(url, cid)
defer muxerWebRTC.Close()
var videoStart bool
noVideo := time.NewTimer(10 * time.Second)
for {
select {
case <-noVideo.C:
log.Println("noVideo")
return
case pck := <-ch:
if pck.IsKeyFrame || AudioOnly {
noVideo.Reset(10 * time.Second)
videoStart = true
}
if !videoStart && !AudioOnly {
continue
}
err = muxerWebRTC.WritePacket(pck)
if err != nil {
log.Println("WritePacket", err)
return
}
}
}
}()
}
4.2 前端修改
对于goland我们首先将.tmpl文件通过右键标记为html格式，然后再修改时就会有前端语法支持和补全支持，便于修改，否则默认是识别为文本的，之后我们修改player.tmpl和app.js，在player.tmpl中添加一些ptz的按钮并通过js与前后端进行数据交互：
player.tmpl:
<html>
<meta http-equiv="Expires" content="0">
<meta http-equiv="Last-Modified" content="0">
<meta http-equiv="Cache-Control" content="no-cache, mustrevalidate">
<meta http-equiv="Pragma" content="no-cache">
<link rel="stylesheet" href="../../static/css/bootstrap.min.css" rel="external nofollow" >
<link rel="stylesheet" href="../../static/css/shanxing.css" rel="external nofollow" >
<script type="text/javascript" src="../../static/js/jquery-3.4.1.min.js"></script>
<script src="../../static/js/bootstrap.min.js"></script>
<script src="../../static/js/adapter-latest.js"></script>
<h2 align=center>
Play Stream {{ .suuid }}<br>
</h2>
<div class="container">
<div class="row">
<div class="col-3">
<div class="list-group">
{{ range .suuidMap }}
<a href="{{ . }}" rel="external nofollow" id="{{ . }}" name="{{ . }}" class="list-group-item list-group-item-action">{{ . }}</a>
{{ end }}
</br>
<div class="sector">
<div class="box s1" id="top" onclick="funTopClick()">
</div>
<div class="box s2" id="right" onclick="funRightClick()">
</div>
<div class="box s3" id="down" onclick="funDownClick()">
</div>
<div class="box s4" id="left" onclick="funLeftClick()">
</div>
<div class="center" id="stop" onclick="funStopClick()">
</div>
</div>
<div class="btn-group">
<button type="button" class="btn btn-default" onclick="funZoomClick(10)">缩放+</button>
<button type="button" class="btn btn-default" onclick="funZoomClick(11)">缩放-</button>
</div>
<div class="btn-group">
<button type="button" class="btn btn-default" onclick="funFocusClick(12)">调焦+</button>
<button type="button" class="btn btn-default" onclick="funFocusClick(13)">调焦-</button>
<button type="button" class="btn btn-default" onclick="funFocusClick(14)">停止调焦</button>
</div>
</div>
</div>
<div class="col">
<input type="hidden" name="suuid" id="suuid" value="{{ .suuid }}">
<input type="hidden" name="port" id="port" value="{{ .port }}">
<input type="hidden" id="localSessionDescription" readonly="true">
<input type="hidden" id="remoteSessionDescription">
<div id="remoteVideos">
<video style="width:600px" id="videoElem" autoplay muted controls></video>
</div>
<div id="div"></div>
</div>
</div>
</div>
<script type="text/javascript" src="../../static/js/app.js?ver={{ .version }}"></script>
</html>
app.js：
let stream = new MediaStream();
let suuid = $('#suuid').val();
let config = {
iceServers: [{
urls: ["stun:stun.l.google.com:19302"]
}]
};
const pc = new RTCPeerConnection(config);
pc.onnegotiationneeded = handleNegotiationNeededEvent;
let log = msg => {
document.getElementById('div').innerHTML += msg + '<br>'
}
pc.ontrack = function(event) {
stream.addTrack(event.track);
videoElem.srcObject = stream;
log(event.streams.length + ' track is delivered')
}
pc.oniceconnectionstatechange = e => log(pc.iceConnectionState)
async function handleNegotiationNeededEvent() {
let offer = await pc.createOffer();
await pc.setLocalDescription(offer);
getRemoteSdp();
}
$(document).ready(function() {
$('#' + suuid).addClass('active');
getCodecInfo();
});
function getCodecInfo() {
$.get("../codec/" + suuid, function(data) {
try {
data = JSON.parse(data);
} catch (e) {
console.log(e);
} finally {
$.each(data,function(index,value){
pc.addTransceiver(value.Type, {
'direction': 'sendrecv'
})
})
}
});
}
let sendChannel = null;
function getRemoteSdp() {
$.post("../receiver/"+ suuid, {
suuid: suuid,
data: btoa(pc.localDescription.sdp)
}, function(data) {
try {
pc.setRemoteDescription(new RTCSessionDescription({
type: 'answer',
sdp: atob(data)
}))
} catch (e) {
console.warn(e);
}
});
}
function ptz(direction) {
$.post("../ptz/", direction, function(data, status){
console.debug("Data: " + data + "nStatus: " + status);
});
}
function funTopClick() {
console.debug("top click");
ptz("action=1")
}
function funDownClick() {
console.debug("down click");
ptz("action=2")
}
function funLeftClick() {
console.debug("left click");
ptz("action=3")
}
function funRightClick() {
console.debug("right click");
ptz("action=4")
}
function funStopClick() {
console.debug("stop click");
ptz("action=9")
}
function funZoomClick(direction) {
console.debug("zoom click"+direction);
ptz("action="+direction)
}
function funFocusClick(direction) {
console.debug("focus click"+direction);
ptz("action="+direction)
}
主要增加了一个扇形按钮和两组按钮组，然后将按钮的点击结合到app.js中进行处理，app.js中则发送post请求调用go后端接口。
4.3 项目结构和编译运行
项目结构如下，部分文件做了备份，实际可以不用：
$tree -a -I ".github|.idea|
build"
.
├── .gitignore
├── CMakeLists.txt
├── Dockerfile
├── LICENSE
├── README.md
├── build.cmd
├── client.c
├── client.h
├── config.go
├── config.json
├── config.json.bak
├── doc
│ ├── demo2.png
│ ├── demo3.png
│ └── demo4.png
├── go.mod
├── go.sum
├── http.go
├── main.go
├── main.go.bak
├── renovate.json
├── soap
│ ├── DeviceBinding.nsmap
│ ├── ImagingBinding.nsmap
│ ├── MediaBinding.nsmap
│ ├── PTZBinding.nsmap
│ ├── PullPointSubscriptionBinding.nsmap
│ ├── RemoteDiscoveryBinding.nsmap
│ ├── custom
│ │ ├── README.txt
│ │ ├── chrono_duration.cpp
│ │ ├── chrono_duration.h
│ │ ├── chrono_time_point.cpp
│ │ ├── chrono_time_point.h
│ │ ├── duration.c
│ │ ├── duration.h
│ │ ├── float128.c
│ │ ├── float128.h
│ │ ├── int128.c
│ │ ├── int128.h
│ │ ├── long_double.c
│ │ ├── long_double.h
│ │ ├── long_time.c
│ │ ├── long_time.h
│ │ ├── qbytearray_base64.cpp
│ │ ├── qbytearray_base64.h
│ │ ├── qbytearray_hex.cpp
│ │ ├── qbytearray_hex.h
│ │ ├── qdate.cpp
│ │ ├── qdate.h
│ │ ├── qdatetime.cpp
│ │ ├── qdatetime.h
│ │ ├── qstring.cpp
│ │ ├── qstring.h
│ │ ├── qtime.cpp
│ │ ├── qtime.h
│ │ ├── struct_timeval.c
│ │ ├── struct_timeval.h
│ │ ├── struct_tm.c
│ │ ├── struct_tm.h
│ │ ├── struct_tm_date.c
│ │ └── struct_tm_date.h
│ ├── dom.c
│ ├── dom.h
│ ├── duration.c
│ ├── duration.h
│ ├── mecevp.c
│ ├── mecevp.h
│ ├── onvif.h
│ ├── smdevp.c
│ ├── smdevp.h
│ ├── soapC.c
│ ├── soapClient.c
│ ├── soapH.h
│ ├── soapStub.h
│ ├── stdsoap2.h
│ ├── stdsoap2_ssl.c
│ ├── struct_timeval.c
│ ├── struct_timeval.h
│ ├── threads.c
│ ├── threads.h
│ ├── typemap.dat
│ ├── wsaapi.c
│ ├── wsaapi.h
│ ├── wsdd.nsmap
│ ├── wsseapi.c
│ └── wsseapi.h
├── stream.go
└── web
├── static
│ ├── css
│ │ ├── bootstrap-grid.css
│ │ ├── bootstrap-grid.css.map
│ │ ├── bootstrap-grid.min.css
│ │ ├── bootstrap-grid.min.css.map
│ │ ├── bootstrap-reboot.css
│ │ ├── bootstrap-reboot.css.map
│ │ ├── bootstrap-reboot.min.css
│ │ ├── bootstrap-reboot.min.css.map
│ │ ├── bootstrap.css
│ │ ├── bootstrap.css.map
│ │ ├── bootstrap.min.css
│ │ ├── bootstrap.min.css.map
│ │ └── shanxing.css
│ └── js
│ ├── adapter-latest.js
│ ├── app.js
│ ├── bootstrap.bundle.js
│ ├── bootstrap.bundle.js.map
│ ├── bootstrap.bundle.min.js
│ ├── bootstrap.bundle.min.js.map
│ ├── bootstrap.js
│ ├── bootstrap.js.map
│ ├── bootstrap.min.js
│ ├── bootstrap.min.js.map
│ └── jquery-3.4.1.min.js
└── templates
├── index.tmpl
└── player.tmpl
8 directories, 111 files
关于cgo和onvif、gSoap部分这里就不多说了，不清楚的可以看前面的总结，gin、bootstramp、jQuery这些也需要一定的前后端概念学习和储备，在其它的分类总结中也零星分布了，不清楚的可以看一下，这里就不再多说了。
编译运行：
GOOS=linux GOARCH=amd64 CGO_ENABLE=1 GO111MODULE=on go run *.go
记得修改一下go.mod中对go版本的依赖，按照cgo的问题，目前至少需要1.18及以上，否则运行ptz可能出现分割违例问题，到我总结这里1.18已经发了正式版本了。
module github.com/deepch/RTSPtoWebRTC go 1.18 require ( github.com/deepch/vdk v0.0.0-20220309163430-c6529706436c github.com/gin-gonic/gin v1.7.7 )
4.4 结果展示
界面效果：
动态调试ptz：
动态调试缩放：
动态调试调焦：
5. 最后
webRTC使用起来几乎感觉不到延迟，但是受制于stun的udp打洞的稳定性，可能会出现卡顿掉线等情况，所以还牵扯到p2p的问题，需要注意这一点，当然，这是远程推流都绕不开的一点，也不算是独有的问题。
加载全部内容