Python视频流到C++服务器

Question

我正在努力学习C++通过项目，我从一个设备流视频显示在另一个设备上。我有一个带有以下客户端代码的PiCamera，它捕获帧并通过TCP流发送它们。

import io 
import socket 
import struct 
import time 
import picamera 


def stream(): 
    servver_ip = "0.0.0.0" #intentionally left out 
    client_socket = socket.socket() 
    client_socket.connect((server_ip, 8123)) 
    connection = client_socket.makefile("wb") 

    try: 
     camera = picamera.PiCamera() 
     camera.resolution = (640, 480) 
     camera.start_preview() 
     time.sleep(2) 
     start = time.time() 
     stream = io.BytesIO() 
     for image in camera.capture_continuous(stream, 'jpeg'): 
      connection.write(struct.pack("<L", stream.tell())) 
      connection.flush() 
      stream.seek(0) 
      connection.write(stream.read()) 
      stream.seek(0) 
      stream.truncate() 
     connection.write(struct.pack("<L", 0)) 
    except Exception as e: 
     raise e 
    finally: 
     connection.close() 
     client_socket.close() 

def main(): 
    while True: 
     try: 
      stream() 
     except ConnectionRefusedError as e: 
      print("Connection refused. It the server on?") 
      time.sleep(2) 
     except Exception as e: 
      print(e) 
      break 

if __name__ == "__main__": 
    main() 

上面的代码直接来自PiCamera recipie，在另一端有一个Python脚本时效果很好。但是，当我尝试接收并显示具有以下C++代码的流时，我只能看到部分帧，并且流动性不佳。有时我会得到0个数据，1个帧，或者一个缓慢的混乱。改变睡眠增加似乎有所改善，但我担心这不是正确的答案。我也尝试在recv中使用MSG_WAITALL标志，但似乎保留了任何数据通过，这可能表明我有一个不正确的值为我的缓冲区大小。

// the standard stuff 
#include <iostream> 
#include <string> 
#include <unistd.h> 

// opencv 
#include <opencv2/opencv.hpp> 
#include <opencv2/highgui/highgui.hpp> 
#include <opencv2/video/video.hpp> 

//socket stuffs 
#include <sys/socket.h> 
#include <sys/types.h> 
#include <netdb.h> 
#include <netinet/in.h> 


int main(int argc, const char * argv[]) { 

    int key; 

    int yes = 1; 

    int height = 480; 
    int width = 640; 

    char* listenPort = "8123"; 

    int bytes = 0; 

    int status; 
    struct addrinfo hints; // define our interface 
    struct addrinfo *serviceinfo; // will point to results 
    struct sockaddr_storage their_addr; 
    memset(&hints, 0, sizeof(hints)); // make sure the struct is empy 
    hints.ai_family = AF_UNSPEC; // don't care if IP4 or IP6 
    hints.ai_socktype = SOCK_STREAM; // TCP 
    hints.ai_flags = AI_PASSIVE; // fill in my IP for me 

    // look up address info for this machine. Will populate service info 
    if ((getaddrinfo(NULL, listenPort, &hints, &serviceinfo)) == -1){ 
     fprintf(stderr, "getaddinfo error: %s\n", gai_strerror(status)); 
     exit(1); 
    } 

    // make the socket 
    int sockfd = socket(serviceinfo->ai_family, serviceinfo->ai_socktype, serviceinfo->ai_protocol); 

    //bind to the correct port 
    if ((bind(sockfd, serviceinfo->ai_addr, serviceinfo->ai_addrlen)) == -1){ 
     fprintf(stderr, "failed to bind: %s\n", gai_strerror(status)); 
     exit(1); 
    } 

    // allow us to reuse the port 
    if ((setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes))) == -1){ 
     perror("setsockopt"); 
     exit(1); 
    } 

    // number of connections to let in the queue 
    int backlog = 1; 

    // start listening for incoming connections 
    if ((listen(sockfd, backlog)) == -1){ 
     perror("listen"); 
     exit(1); 
    } 

    // start accepting incoming connections 
    socklen_t addr_size = sizeof(their_addr); 
    int new_fd = accept(sockfd, (struct sockaddr *)&their_addr, &addr_size); // use this for all send and receive calls 
    if (new_fd == -1){ 
     perror("accept"); 
     exit(1); 
    } 

    // empty image object 
    cv::Mat img = cv::Mat::zeros(height, width, CV_8UC1); 

    if (!img.isContinuous()){ 
     img = img.clone(); 
    } 

    int image_size = img.total() * img.elemSize(); 

    cv::Mat rawImage = cv::Mat::zeros(1, image_size, CV_8UC1); 

    if (!rawImage.isContinuous()){ 
     rawImage = rawImage.clone(); 
    } 
    cv::Mat flipped = cv::Mat::zeros(height, width, CV_8UC1); 

    std::cout << "Press q to quit" << std::endl; 
    cv::namedWindow("Sentry", cv::WINDOW_AUTOSIZE); 

    while (key != 'q'){ 
     std::cout << "Capturing" << std::endl; 

     if ((bytes = recv(new_fd, rawImage.data, image_size, 0)) == -1){ 
      perror("Failed to receive bytes!"); 
      exit(1); 
     } 

     img = cv::imdecode(rawImage, CV_LOAD_IMAGE_COLOR); 
     cv::flip(img, flipped, 1); 


     if (flipped.data != NULL) { 
      // show image. using a global here 
      cv::imshow("Sentry", flipped); 
     } 

     memset(rawImage.data, 0x0, sizeof(rawImage)); 

     // look for quit key 
     key = cv::waitKey(10); 

     // pause for half a second 
     usleep(500000); 
    }; 

    cv::destroyAllWindows(); 

    freeaddrinfo(serviceinfo); // clear the linked list 

    close(sockfd); 

    return 0; 

} 

我正在寻找任何提示，答案或只是指向正确的方向。提前致谢。

编辑：工作解决方案
首先，感谢凯文帮助我。我的问题是我没有意识到我的初始Python客户端正在发送图像大小。做了一些搜索和处理下面的答案，我开始抓住前4个字节，并开始调整我的cv::Mat rawImage的大小。而不是编写逻辑来检查recv以确保我获得所有数据，我正在使用MSG_WAITALL。该标志结合获得正确的有效载荷大小使一切顺利进行。非常谦逊的经历。

#define CHUNKSIZE 500 

int main(int argc, const char * argv[]) { 

    // skipping code from before 

    int32_t image_size = 0; 
    cv::Mat rawImage; 
    long data_received = 0; 
    long success; 

    cv::namedWindow("Sentry", cv::WINDOW_AUTOSIZE); 

    while (key != 'q'){ 
     std::cout << "Capturing" << std::endl; 

     // Very important!! Grab the image_size here 
     success = recv(new_fd, &image_size, 4 , NULL); 
     if (success == -1){ 
      perror("Failed to receive file size!"); 
      exit(1); 
     } 
     // if your image size is extremely large, it's probably 
     // because you're grabing the wrong number of bytes above 
     if (image_size > 300000){ 
      std::cout << "Image size is too large " << image_size << std::endl; 
      exit(1); 
     } 

     if (image_size > 0) { 

      // 
      rawImage = cv::Mat::zeros(1, image_size, CV_8UC1); 
      success = recv(new_fd, rawImage.data, image_size, MSG_WAITALL); 
      if (success == -1){ 
       perror("Failed to receive"); 
       exit(1); 
      } 

      img = cv::imdecode(rawImage, CV_LOAD_IMAGE_COLOR); 

      if (img.data != NULL) { 
       // show image. using a global here 
       cv::imshow("Sentry", img); 
      } else { 
       std::cout << "Image is null!" << std::endl; 
      } 

      memset(&rawImage, 0x0, sizeof(rawImage)); 

      // look for quit key 
      key = cv::waitKey(10); 
     } else { 
      std::cout << "No message yet" << std::endl; 

     } 
     image_size = 0; 
     // pause for half a second 
     usleep(500000); 
    }; 

    cv::destroyAllWindows(); 

    freeaddrinfo(serviceinfo); // clear the linked list 

    close(sockfd); 

    return 0; 

} 

Answer 1

当我使用C++创建一个实时流Raspberry Pi到我的电脑时，我也有同样的问题。

我解决了它只发送图像的块。测试块大约500字节，然后增加它，一旦你看到它的工作。不要发送整个文件，否则程序只会得到帧的一部分。

另外，不要像@Sam那样读取send（）或recv（）函数的返回码，我实际上发现它在创建实时流时不可靠。只需将它发送出去，应该没问题。

我有我的源代码，如果你想看看它，我知道很难创建一个使用C++的实时流服务器，因为几乎没有文档，所以我想帮助你，因为它给了我太多的头痛！让我知道如果你想源代码祝你好运！

客户端应该看起来像这样，这不会编译！我试图使它易于阅读：

//Client  
#define CHUNK_SIZE 500 

int main() 
{ 
    //do all the connection stuff here and connect to it 
    int filesize; 
    int dataReceived = 0; 
    char received_message[10]; 
    std::vector<char> imageData; 
    while (1) 
    { 
     recv(Connection, received_message, 11, NULL); //recieved File size of image 
     filesize = atoi(received_message); //Turning File size char into an integer 

     imageData.resize(filesize); //Setting up vector to recieve image data 

     while (dataReceived < filesize) //keep running until we get enough bytes 
     { 
      if (filesize - dataReceived >= CHUNK_SIZE) //if the amount of bytes left to recieve is greater than or equal to chunk size, then lets receive a CHUNK_SIZE of the image 
      { 
       recv(Connection, &imageData[dataReceived], CHUNK_SIZE, 0); 
       dataReceived += CHUNK_SIZE; 
      } 
      else //Else, if the amount of bytes left to recieve is smaller than the CHUNK_SIZE then lets receive a specific amount of bytes 
      { 
       recv(Connection, &imageData[dataReceived], fileSize - dataReceived, 0); 
       dataReceived += fileSize - dataReceived; 
      } 
     Sleep(Amount of sleep); //Note, python code will always be slower than C++ so you must set a Sleep() function. 
     } 
     std::cout << "Got the image, it is stored in the imageData vector!" << std::endl; 
     std::cout << "Press enter to get another image!" << std::endl; 
     std::cin.get(); 

    } 
} 

Answer 2

显示的代码大多忽略recv()的返回值。

if ((bytes = recv(new_fd, rawImage.data, image_size, 0)) == -1){ 

确实，错误导致recv()返回-1。

但是如果recv()没有返回-1，这并不意味着recv()已经读取image_size字节，在这里。

但是现在显示的代码似乎假设已经读取了整个图像，即image_size字节。这是一个错误的假设。当涉及网络套接字时，如果接收方迄今收到的字节数较少，recv()将立即返回。

来自recv()的正返回值指示实际接收并放入缓冲区的字节数。如果字节数少于预期，则需要重新拨打recv()。需要注意的是，你不能只是

recv(new_fd, rawImage.data, image_size, 0) 

再次，由于rawImage.data初始部分已经包含然而，许多字节被第一次调用接收recv()。在这里，您需要制定一些简单的逻辑来阅读其余部分（请记住，第二次致电recv()的电话不保证读取其余部分）。