對最近的RTP和H264學習進行總結整理-04.20，rtp-04.20
雖然還是沒有搞出來，但總感覺快了哈哈（哪來的自信）
1、RTP協議接受數據
#region 1-RTP協議變量聲明
RTPSession session;
RTPReceiver receiver;
RTPParticipant participant;
private Dictionary<uint, List<RTPPacket>> Clients;
#endregion
#region 對RTP進行初始化,並接收數據,調用之後就可以接收數據了
session = new RTPSession();
receiver = new RTPReceiver();
IPEndPoint rtpEp = new IPEndPoint(IPAddress.Parse("192.168.1.109"), 5000);
participant = new RTPParticipant(rtpEp);
receiver.AddParticipant(participant);
session.NewRTPPacket = new RTPSession.NewRTPPacket_Callback(NewRTPPacket);
session.AddReceiver(receiver);
Clients = new Dictionary<uint, List<RTPPacket>>();
#endregion
其中NewRTPPackt是
public delegate bool NewRTPPacket_Callback(
RTPPacket packet
)
類型的委托。packet為接收到的RTP包，我們就對這些包進行處理得到想要的幀，然後再把幀進行解碼，得到想要的圖像（我是這樣理解的）
2、H.264進行解碼
我從網絡上搜索到了一個海思的DLL,可以對H.264進行解碼
#region 解碼器相關變量聲明
/// <summary>
/// 數據的句柄
/// </summary>
IntPtr pData;
/// <summary>
/// 這是解碼器屬性信息
/// </summary>
public H264Dec.hiH264_DEC_ATTR_S decAttr;
/// <summary>
/// 這是解碼器輸出圖像信息
/// </summary>
public H264Dec.hiH264_DEC_FRAME_S _decodeFrame = new H264Dec.hiH264_DEC_FRAME_S();
/// <summary>
/// 解碼器句柄
/// </summary>
public IntPtr _decHandle;
#endregion
#region 解碼器相關初始化，一般在窗口load中進行初始化
decAttr = new H264Dec.hiH264_DEC_ATTR_S();
decAttr.uPictureFormat = 0;
decAttr.uStreamInType = 0;
decAttr.uPicWidthInMB = 480 >> 4;
decAttr.uPicHeightInMB = 640 >> 4;
decAttr.uBufNum = 8;
decAttr.uWorkMode = 16;
//創建、初始化解碼器句柄
_decHandle = H264Dec.Hi264DecCreate(ref decAttr);
//_decodeFrame = new H264Dec.hiH264_DEC_FRAME_S();
#endregion
//這一寫代碼就是h264解碼的代碼，其中未聲明的函數和變量會在下面進行聲明給出，主要是講YUV轉為RGB，在保存為Bitmap文件
if (H264Dec.Hi264DecAU(_decHandle, pData, (uint)newData.Length, 0, ref _decodeFrame, 0) == 0)
{
if (_decodeFrame.bError == 0)
{
//策畫 y u v 的長度
var yLength = _decodeFrame.uHeight * _decodeFrame.uYStride;
var uLength = _decodeFrame.uHeight * _decodeFrame.uUVStride / 2;
var vLength = uLength;
var yBytes = new byte[yLength];
var uBytes = new byte[uLength];
var vBytes = new byte[vLength];
var decodedBytes = new byte[yLength + uLength + vLength];
//_decodeFrame 是解碼後的數據對象,裡面包含 YUV 數據、寬度、高度等信息
Marshal.Copy(_decodeFrame.pY, yBytes, 0, (int)yLength);
Marshal.Copy(_decodeFrame.pU, uBytes, 0, (int)uLength);
Marshal.Copy(_decodeFrame.pV, vBytes, 0, (int)vLength);
//將從 _decodeFrame 中取出的 YUV 數據放入 decodedBytes 中
Array.Copy(yBytes, decodedBytes, yLength);
Array.Copy(uBytes, 0, decodedBytes, yLength, uLength);
Array.Copy(vBytes, 0, decodedBytes, yLength + uLength, vLength);
ConvertYUV2RGB(yuv, rgb, width, height);
ConvertYUV2RGB(decodedBytes, rgb, width, height);
// 寫 BMP 文件。
WriteBMP(rgb, width, height, string.Format("E:\\test\\yuv2bmp_{0}.bmp", index++));
}
}
其中pData為需要的一幀數據，因為pData為Intptr類型，而一幀數據是byte[]類型，所以我從網上查了查怎麼轉換，下面是代碼，newData是byte【】，pData是intptr類型。
GCHandle hObject = GCHandle.Alloc(newData, GCHandleType.Pinned);
pData = hObject.AddrOfPinnedObject();
H264解碼類
![]()
public class H264Dec
{
public const int HI_SUCCESS = 0;
public const int HI_FAILURE = -1;
public const int HI_LITTLE_ENDIAN = 1234;
public const int HI_BIG_ENDIAN = 4321;
public const int HI_DECODER_SLEEP_TIME = 60000;
public const int HI_H264DEC_OK = 0;
public const int HI_H264DEC_NEED_MORE_BITS = -1;
public const int HI_H264DEC_NO_PICTURE = -2;
public const int HI_H264DEC_ERR_HANDLE = -3;
[DllImport("hi_h264dec_w.dll", EntryPoint = "Hi264DecImageEnhance", CallingConvention = CallingConvention.Cdecl)]
public static extern int Hi264DecImageEnhance(IntPtr hDec, ref hiH264_DEC_FRAME_S pDecFrame, uint uEnhanceCoeff);
[DllImport("hi_h264dec_w.dll", EntryPoint = "Hi264DecCreate", CallingConvention = CallingConvention.Cdecl)]
public static extern IntPtr Hi264DecCreate(ref hiH264_DEC_ATTR_S pDecAttr);
[DllImport("hi_h264dec_w.dll", EntryPoint = "Hi264DecDestroy", CallingConvention = CallingConvention.Cdecl)]
public static extern void Hi264DecDestroy(IntPtr hDec);
[DllImport("hi_h264dec_w.dll", EntryPoint = "Hi264DecGetInfo", CallingConvention = CallingConvention.Cdecl)]
public static extern int Hi264DecGetInfo(ref hiH264_LIBINFO_S pLibInfo);
/// <summary>
/// 對輸入的一段碼流進行解碼並按幀輸出圖像
/// </summary>
/// <param name="hDec">解碼器句柄</param>
/// <param name="pStream">碼流起始地址</param>
/// <param name="iStreamLen">碼流長度</param>
/// <param name="ullPTS">時間戳信息</param>
/// <param name="pDecFrame">圖像信息</param>
/// <param name="uFlags">解碼模式 0：正常解碼；1、解碼完畢並要求解碼器輸出殘留圖像</param>
/// <returns></returns>
[DllImport("hi_h264dec_w.dll", EntryPoint = "Hi264DecFrame", CallingConvention = CallingConvention.Cdecl)]
public static extern int Hi264DecFrame(IntPtr hDec, IntPtr pStream, uint iStreamLen, ulong ullPTS, ref hiH264_DEC_FRAME_S pDecFrame, uint uFlags);
[DllImport("hi_h264dec_w.dll", EntryPoint = "Hi264DecAU", CallingConvention = CallingConvention.Cdecl)]
public static extern int Hi264DecAU(IntPtr hDec, IntPtr pStream, uint iStreamLen, ulong ullPTS, ref hiH264_DEC_FRAME_S pDecFrame, uint uFlags);
/// <summary>
/// 解碼器屬性信息。
/// </summary>
[StructLayout(LayoutKind.Sequential)]
public struct hiH264_DEC_ATTR_S
{
/// <summary>
/// 解碼器輸出圖像格式，目前解碼庫只支持YUV420圖像格式
/// </summary>
public uint uPictureFormat;
/// <summary>
/// 輸入碼流格式 0x00: 目前解碼庫只支持以“00 00 01”為nalu分割符的流式H.264碼流
/// </summary>
public uint uStreamInType;
/// <summary>
/// 圖像寬度
/// </summary>
public uint uPicWidthInMB;
/// <summary>
/// 圖像高度
/// </summary>
public uint uPicHeightInMB;
/// <summary>
/// 參考幀數目
/// </summary>
public uint uBufNum;
/// <summary>
/// 解碼器工作模式
/// </summary>
public uint uWorkMode;
/// <summary>
/// 用戶私有數據
/// </summary>
public IntPtr pUserData;
/// <summary>
/// 保留字
/// </summary>
public uint uReserved;
}
/// <summary>
/// 解碼器輸出圖像信息數據結構
/// </summary>
[StructLayout(LayoutKind.Sequential)]
public struct hiH264_DEC_FRAME_S
{
/// <summary>
/// Y分量地址
/// </summary>
public IntPtr pY;
/// <summary>
/// U分量地址
/// </summary>
public IntPtr pU;
/// <summary>
/// V分量地址
/// </summary>
public IntPtr pV;
/// <summary>
/// 圖像寬度(以像素為單位)
/// </summary>
public uint uWidth;
/// <summary>
/// 圖像高度(以像素為單位)
/// </summary>
public uint uHeight;
/// <summary>
/// 輸出Y分量的stride (以像素為單位)
/// </summary>
public uint uYStride;
/// <summary>
/// 輸出UV分量的stride (以像素為單位)
/// </summary>
public uint uUVStride;
/// <summary>
/// 圖像裁減信息:左邊界裁減像素數
/// </summary>
public uint uCroppingLeftOffset;
/// <summary>
/// 圖像裁減信息:右邊界裁減像素數
/// </summary>
public uint uCroppingRightOffset;
/// <summary>
/// 圖像裁減信息:上邊界裁減像素數
/// </summary>
public uint uCroppingTopOffset;
/// <summary>
/// 圖像裁減信息:下邊界裁減像素數
/// </summary>
public uint uCroppingBottomOffset;
/// <summary>
/// 輸出圖像在dpb中的序號
/// </summary>
public uint uDpbIdx;
/// <summary>
/// 圖像類型：0:幀; 1:頂場; 2:底場 */
/// </summary>
public uint uPicFlag;
/// <summary>
/// 圖像類型：0:幀; 1:頂場; 2:底場 */
/// </summary>
public uint bError;
/// <summary>
/// 圖像是否為IDR幀：0:非IDR幀;1:IDR幀
/// </summary>
public uint bIntra;
/// <summary>
/// 時間戳
/// </summary>
public ulong ullPTS;
/// <summary>
/// 圖像信號
/// </summary>
public uint uPictureID;
/// <summary>
/// 保留字
/// </summary>
public uint uReserved;
/// <summary>
/// 指向用戶私有數據
/// </summary>
public IntPtr pUserData;
}
/// <summary>
/// 解碼庫版本、版權和能力集信息。
/// </summary>
[StructLayout(LayoutKind.Sequential)]
public struct hiH264_LIBINFO_S
{
/// <summary>
/// 主編號
/// </summary>
public uint uMajor;
/// <summary>
/// 次編號
/// </summary>
public uint uMinor;
/// <summary>
/// 發布編號
/// </summary>
public uint uRelease;
/// <summary>
/// 建構編號
/// </summary>
public uint uBuild;
/// <summary>
/// 版本信息
/// </summary>
[MarshalAs(UnmanagedType.LPStr)]
public string sVersion;
/// <summary>
/// 版權信息
/// </summary>
[MarshalAs(UnmanagedType.LPStr)]
public string sCopyRight;
/// <summary>
/// 解碼庫能力集
/// </summary>
public uint uFunctionSet;
/// <summary>
/// 支持的輸出圖像格式
/// </summary>
public uint uPictureFormat;
/// <summary>
/// 輸入碼流格式
/// </summary>
public uint uStreamInType;
/// <summary>
/// 最大圖像寬度(以像素為單位)
/// </summary>
public uint uPicWidth;
/// <summary>
/// 最大圖像高度(以像素為單位)
/// </summary>
public uint uPicHeight;
/// <summary>
/// 最大參考幀數目
/// </summary>
public uint uBufNum;
/// <summary>
/// 保留字
/// </summary>
public uint uReserved;
}
/// <summary>
/// 用戶私有數據信息。
/// </summary>
[StructLayout(LayoutKind.Sequential)]
public struct hiH264_USERDATA_S
{
/// <summary>
/// 用戶數據類型
/// </summary>
public uint uUserDataType;
/// <summary>
/// 用戶數據長度
/// </summary>
public uint uUserDataSize;
/// <summary>
/// 用戶數據緩沖區
/// </summary>
public IntPtr pData;
/// <summary>
/// 指向下一段用戶數據
/// </summary>
public IntPtr pNext;
}
}
View Code
這是YUV轉RGB圖像。
![]()
/// <summary>
/// 將轉換後的 RGB 圖像數據按照 BMP 格式寫入文件。
/// </summary>
/// <param name="rgbFrame">RGB 格式圖像數據。</param>
/// <param name="width">圖像寬（單位：像素）。</param>
/// <param name="height">圖像高（單位：像素）。</param>
/// <param name="bmpFile"> BMP 文件名。</param>
static void WriteBMP(byte[] rgbFrame, int width, int height, string bmpFile)
{
// 寫 BMP 圖像文件。
int yu = width * 3 % 4;
int bytePerLine = 0;
yu = yu != 0 ? 4 - yu : yu;
bytePerLine = width * 3 + yu;
using (FileStream fs = File.Open(bmpFile, FileMode.Create))
{
using (BinaryWriter bw = new BinaryWriter(fs))
{
bw.Write('B');
bw.Write('M');
bw.Write(bytePerLine * height + 54);
bw.Write(0);
bw.Write(54);
bw.Write(40);
bw.Write(width);
bw.Write(height);
bw.Write((ushort)1);
bw.Write((ushort)24);
bw.Write(0);
bw.Write(bytePerLine * height);
bw.Write(0);
bw.Write(0);
bw.Write(0);
bw.Write(0);
byte[] data = new byte[bytePerLine * height];
int gIndex = width * height;
int bIndex = gIndex * 2;
for (int y = height - 1, j = 0; y >= 0; y--, j++)
{
for (int x = 0, i = 0; x < width; x++)
{
data[y * bytePerLine + i++] = rgbFrame[bIndex + j * width + x]; // B
data[y * bytePerLine + i++] = rgbFrame[gIndex + j * width + x]; // G
data[y * bytePerLine + i++] = rgbFrame[j * width + x]; // R
}
}
bw.Write(data, 0, data.Length);
bw.Flush();
}
}
}
/// <summary>
/// 將一桢 YUV 格式的圖像轉換為一桢 RGB 格式圖像。
/// </summary>
/// <param name="yuvFrame">YUV 格式圖像數據。</param>
/// <param name="rgbFrame">RGB 格式圖像數據。</param>
/// <param name="width">圖像寬（單位：像素）。</param>
/// <param name="height">圖像高（單位：像素）。</param>
static void ConvertYUV2RGB(byte[] yuvFrame, byte[] rgbFrame, int width, int height)
{
int uIndex = width * height;
int vIndex = uIndex + ((width * height) >> 2);
int gIndex = width * height;
int bIndex = gIndex * 2;
int temp = 0;
for (int y = 0; y < height; y++)
{
for (int x = 0; x < width; x++)
{
// R分量
temp = (int)(yuvFrame[y * width + x] + (yuvFrame[vIndex + (y / 2) * (width / 2) + x / 2] - 128) * YUV2RGB_CONVERT_MATRIX[0, 2]);
rgbFrame[y * width + x] = (byte)(temp < 0 ? 0 : (temp > 255 ? 255 : temp));
// G分量
temp = (int)(yuvFrame[y * width + x] + (yuvFrame[uIndex + (y / 2) * (width / 2) + x / 2] - 128) * YUV2RGB_CONVERT_MATRIX[1, 1] + (yuvFrame[vIndex + (y / 2) * (width / 2) + x / 2] - 128) * YUV2RGB_CONVERT_MATRIX[1, 2]);
rgbFrame[gIndex + y * width + x] = (byte)(temp < 0 ? 0 : (temp > 255 ? 255 : temp));
// B分量
temp = (int)(yuvFrame[y * width + x] + (yuvFrame[uIndex + (y / 2) * (width / 2) + x / 2] - 128) * YUV2RGB_CONVERT_MATRIX[2, 1]);
rgbFrame[bIndex + y * width + x] = (byte)(temp < 0 ? 0 : (temp > 255 ? 255 : temp));
}
}
}
View Code
3、這可能就是我遇到問題的地方了，怎麼把RTPPack中的包數據轉換為一幀圖像信息，我找到的資料是;
#region 對收到的數據進行處理
if (!Clients.ContainsKey(packet.SSRC))//如果接受端第一次接受到某源的數據，則加入到
{
if (Clients.Count < 4)//如果發送端為4，則丟棄包
{
Clients.Add(packet.SSRC, new List<RTPPacket> { packet });
//ImagesBoxMapping[ImagesBoxMapping.First(pair => pair.Value == null).Key] = packet.SSRC;
}
}
else
{
Clients[packet.SSRC].Add(packet);
}
if (packet.Marker)//如果已經發送完畢
{
//丟包檢測
var orderPackets = Clients[packet.SSRC].OrderBy(rtpPacket => rtpPacket.SequenceNumber);
if (Clients[packet.SSRC].Count != (orderPackets.Last().SequenceNumber - orderPackets.First().SequenceNumber + 1))
{
Clients[packet.SSRC].Clear();//清空緩存區
return true;
}
//1.包重組
var count = Clients[packet.SSRC].Sum(rtpPacket => rtpPacket.DataSize);//數據總數
var newData = new byte[count];
long offSet = 0;
foreach (var rtpPacket in Clients[packet.SSRC])
{
Array.Copy(rtpPacket.DataPointer, 0, newData, offSet, rtpPacket.DataSize);
offSet += rtpPacket.DataSize;
}
Clients[packet.SSRC].Clear();//清空緩存區
這裡我理解的是newData裡面就是一幀數據，但我測試了一下不對（暈）。
4、總結
這幾天一直想要盡快做出來，卻總沒有辦法深入去研究視頻方面的東西。比如得到的包怎麼變為一幀，怎麼從一幀裡面提取需要的數據，什麼PPS、SPS、IDR都是什麼，雖然知道名詞，但總沒法很明確的說出來。
我的解碼思路是：RTP協議收到包後（這一步沒有問題），將包的數據轉為幀（這個地方可能出問題了，也可能是傳過來的幀數據不符合解碼的要求），再把一幀的數據傳給H264解碼類解碼，解碼後輸出的是YUV，YUV->RGB->圖片進行顯示就可以了。這是我的思路，但沒有成功。如果讀者您懂這一方面，還希望給我指導。謝謝
每天寫一點點，就能進步一點點.
