[C ＃ -python communication] C ＃ transmits images to Python server through UDP communication for target detection (using pre training yolov4 model)

Screenshot

Demo screen

Pre training model file

note ：

One 、udp client C# class

using System;
using System.IO;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Net;
using System.Net.Sockets;
using System.Drawing;
using System.Text.Json;
namespace WindowsFormsApp1
{
public class Upd
{
private IPAddress serverIP = IPAddress.Parse("127.0.0.1");//192.168.0.100
private static int max_size = 65000;// Send a single packet size
private UdpClient udpClient = new UdpClient(11000);
private class Packs_Info
{
public int packs_num { get; set; }
}
public byte[] ImageToByte(Image img)
{
using (var stream = new MemoryStream())
{
img.Save(stream, System.Drawing.Imaging.ImageFormat.Png);
return stream.ToArray();
}
}
// Send images
public void SendData(Image img)
{
try
{
udpClient.Connect(serverIP, 11000);
var buffer = ImageToByte(img);// Images
var num_packs = 1;
if (buffer.Length > max_size)
{
num_packs = (int)Math.Ceiling((double)buffer.Length / max_size);
}
var packs_info = new Packs_Info { packs_num = num_packs };// Package information
var packs_info_json = JsonSerializer.Serialize<Packs_Info>(packs_info);// Package information serialization
//var packs_info_Bytes = Encoding.ASCII.GetBytes(packs_info_json);
var packs_info_Bytes = Encoding.ASCII.GetBytes(packs_info_json);
udpClient.Send(packs_info_Bytes, packs_info_Bytes.Length);// Send packet information
var left = 0;
var right = max_size;
if (num_packs == 1)// The number of bytes occupied by a single image is less than max_size
{
udpClient.Send(buffer, buffer.Length);// Send images at once
Console.WriteLine("package number: {0} , Size: {1} ", num_packs, buffer.Length);
}
else// Multiple packets form an image
{
for (int i = 0; i < num_packs; i++)// Send a single image in multiple packets
{
Console.WriteLine("left : {0} , right : {1}", left, right);
var temp = 0;
var data = new byte[max_size];
for (int j = left; j < right; j++)
{
if (j == buffer.Length)
break;
data[temp] = buffer[j];
temp++;
}
left = right;// Update the starting point of the next package
right += max_size;// The end of the next package
udpClient.Send(data, data.Length);// Send a package
Console.WriteLine("package number: {0} , Size: {1} ", i, data.Length);
}
}
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
}
}
// receive data ： Identify the result string
public string ReceiveData()
{
var RemoteIpEndPoint = new IPEndPoint(serverIP, 11000);
var receiveBytes = udpClient.Receive(ref RemoteIpEndPoint);
// var jsonString = Encoding.ASCII.GetString(receiveBytes);
var jsonString = Encoding.ASCII.GetString(receiveBytes);
var box_info = JsonSerializer.Deserialize<Box_Info>(jsonString);
Console.WriteLine("box {0} , color: {1} ", box_info.box, box_info.color);
return jsonString;
//var box_info = JsonSerializer.Deserialize<Box_Info>(jsonString);
//Console.WriteLine("box {0} , color: {1} ", box_info.box, box_info.color);
}
private class Box_Info
{
public string name { get; set; }//
public int[] box { get; set; }// Frame x,y,w,h Array
public string label { get; set; }// label
public int[] color { get; set; }// Border color array
}// Border information
public string stopStream()
{
udpClient.Close();
return "stoped";
}
}
}

Two 、C# Main form class code

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Threading;
using System.IO;
using System.Windows.Forms;
using Emgu.CV;
using System.Text.Json;
using Emgu.CV.Structure;
using Emgu.CV.CvEnum;
namespace WindowsFormsApp1
{
public partial class Form1 : Form
{
private VideoCapture objCapture;// Video capture
private Mat _frame;
private Mat _frame_before;
private bool isOpen = true;// Turn on the video stream
private bool is_stream = true;
private int Fps;// Frame rate
private Thread thread_stream;//udp Send image thread
private Thread thread_receive;// Receiving thread
public string box_info_str;// Border information string private
public static object lockObj = new Object();// lock
private Upd udp;
private class Box_Info
{
public string name { get; set; }//
public int[] box { get; set; }// Frame x,y,w,h Array
public string label { get; set; }// label
public int[] color { get; set; }// Border color array
}// Border information
public Form1()
{
InitializeComponent();
}
// Process images
private void ProcessFrame(object sender, EventArgs e)
{
if (objCapture != null && objCapture.Ptr != IntPtr.Zero)
{
objCapture.Retrieve(_frame_before, 0);// Read a frame ->_frame_before
Fps = (int)objCapture.Get(CapProp.Fps);// Get frame rate
CvInvoke.Flip(_frame_before, _frame, FlipType.Horizontal);// Invert the image horizontally
if (box_info_str != null)
{
var box_info = JsonSerializer.Deserialize<Box_Info>(box_info_str);// Deserialize border information
var rectangle = new Rectangle(box_info.box[0], box_info.box[1], box_info.box[2], box_info.box[3]);// Frame
var mcvScalar = new MCvScalar(box_info.color[0], box_info.color[1], box_info.color[2]);// Color
CvInvoke.Rectangle(_frame, rectangle, mcvScalar, 2);// bound box
// Show labels
CvInvoke.PutText(_frame, box_info.label, new Point(box_info.box[0], box_info.box[1] - 10), FontFace.HersheyComplexSmall, 1, mcvScalar, 2);
}
// Display frame rate
CvInvoke.PutText(_frame, Fps.ToString(), new Point(0, 25), FontFace.HersheyComplexSmall, 1.5, new MCvScalar(0, 0, 255), 2);
Mat temp = new Mat();
_frame.CopyTo(temp);
CvInvoke.Resize(temp, temp, pictureBox1.Size);
pictureBox1.Image = temp.ToBitmap();// Display images
temp.Dispose();
Thread.Sleep(3000 / (Fps));
}
}
private void button1_Click(object sender, EventArgs e)
{
if (isOpen)
{
button1.Text = "Trun Off Camera";
objCapture = new VideoCapture("test.mp4");// Open the test video
objCapture.ImageGrabbed += ProcessFrame;// Capture frame event handler
_frame = new Mat();
_frame_before = new Mat();
if (objCapture != null)
objCapture.Start();// Start playing
isOpen = false;
}
else
{
button1.Text = "Trun On Camera";
objCapture.Stop();// Stop playing
isOpen = true;
}
// isOpen = !isOpen;
}
private void button2_Click(object sender, EventArgs e)
{
if (is_stream)
{
udp = new Upd();// Instantiation udp
button2.Text = "Stream Stop";
thread_stream = new Thread(stream);// Send video stream processing function
thread_receive = new Thread(receive);// receive python Returned data
thread_stream.Start();
thread_receive.Start();
}
else
{
thread_stream.Abort();
thread_receive.Abort();
udp.stopStream();// close udp client
button2.Text = "Stream Start";
}
is_stream = !is_stream;
}
private void stream()
{
try
{
while (true)
{
udp.SendData(_frame_before.ToBitmap());// Send video images frame by frame
Thread.Sleep(3000 / (Fps));
}
}
catch (ThreadAbortException e)
{
Console.WriteLine("Stream Stopping");
}
finally
{
Console.WriteLine("Stream Stopped");
}
}
private void receive()
{
try
{
while (true)
{
lock (lockObj)
{
box_info_str = udp.ReceiveData();// Receive the returned target detection information
}
Console.WriteLine(box_info_str);
}
}
catch (ThreadAbortException e)
{
Console.WriteLine("Receive Stopping");
}
finally
{
Console.WriteLine("Receive Stopped");
}
}
}
}

3、 ... and 、Python udp Server side

import socket
import json
import numpy as np
#import cv2
from cv2 import cv2
import queue
import time
HOST = '127.0.0.1'#192.168.0.100
PORT = 11000
Max_size = 65000
packs_info = None
frame = None
class_names = []
frame_data = None
CONFIDENCE_THRESHOLD = 0.2
NMS_THRESHOLD = 0.4
COLORS = [(0, 255, 255), (255, 255, 0), (0, 255, 0), (255, 0, 0)]
frame_buffer = queue.Queue()
#net = cv2.dnn.readNet('model/yolov4-tiny.cfg','model/yolov4-tiny.weights')
net = cv2.dnn.readNet('model/yolov4.cfg','model/yolov4.weights')
#net = cv2.dnn.readNet('model/yolov4-custom.cfg','model/yolov4-custom.weights')
net.setPreferableBackend(cv2.dnn.DNN_BACKEND_CUDA)
net.setPreferableTarget(cv2.dnn.DNN_TARGET_CUDA)#_FP16
model = cv2.dnn_DetectionModel(net)
model.setInputParams(size=(416, 416),scale=1/255,swapRB=True)
#model.setInputParams(size=(608, 608),scale=1/255,swapRB=True)#size=(608, 608)
with open("model/coco_classes.txt", "r") as f:
class_names = [cname.strip() for cname in f.readlines()]
s = socket.socket(socket.AF_INET,socket.SOCK_DGRAM)
s.bind((HOST, PORT))
print('server start at: %s:%s' % (HOST, PORT))
print('wait for connection...')
def img_decode(indata):
_frame = np.frombuffer(indata, dtype=np.uint8)
_frame = _frame.reshape(_frame.shape[0], 1)
_frame = cv2.imdecode(_frame, cv2.IMREAD_COLOR)
_frame = cv2.flip(_frame, 1)
return _frame
while True:
indata, addr = s.recvfrom(Max_size)
print('recvfrom ' + str(addr) + ': ' + str(len(indata)))
#check package is information or image data
if len(indata) < 50:
packs_info = json.loads(indata.decode())
print(packs_info)
else:
indata_buffer = []
indata_buffer.append(indata)
if packs_info:
package_size = packs_info['packs_num']
if package_size == 1:
frame = img_decode(indata_buffer[0])
frame_buffer.put(frame)
#combine the image data, whilch is oversize
else:
frame_data = None
waiting_num = 0
start = 0
end = 0
run_time = 0
while True:
start = time.process_time()
indata, addr = s.recvfrom(Max_size)
if len(indata) > 50:
print('recvfrom2 ' + str(addr) + ': ' + str(len(indata)))
indata_buffer.append(indata)
if len(indata_buffer) == package_size:
for i in indata_buffer:
if frame_data is None:
frame_data = i
else:
frame_data += i
frame = img_decode(frame_data)
frame_buffer.put(frame)
break
end = time.process_time()
run_time += start-end
if run_time > 0.1 :
break
#predict and send to client
if frame_buffer.empty() is False and frame_buffer.qsize() > 1:
try:
classes, scores, boxes = model.detect(frame_buffer.get(), CONFIDENCE_THRESHOLD, NMS_THRESHOLD)
for (classid, score, box) in zip(classes, scores, boxes):
color = COLORS[int(classid) % len(COLORS)]
label = "%s : %f" % (class_names[int(classid)], score)
cv2.rectangle(frame, box, color, 2)
cv2.putText(frame, label, (box[0], box[1] - 10), cv2.FONT_HERSHEY_SIMPLEX, 1, color, 2)
tempList = {"name":"box_info","box":box.tolist(),"label":label,"color":color}
json_str = json.dumps(tempList)
s.sendto(json_str.encode(), addr)# There is a problem sending data
packs_info = None
except:
print("pass")
if frame is not None and type(frame) == np.ndarray:
cv2.imshow("Stream", frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
                    break

The End