C++ 套接字 accept() get 资源死锁避免了 epoll 的 errno

Question

我是一个学习I/O多路复用的C++初学者。

这是我的代码：

test.cpp(不使用epoll()并且效果很好)：

#include <iostream>
#include <cstdio>
#include <netinet/in.h>
#include <unistd.h>
#include <string.h>
int main() {
std::cout << "Hello" << std::endl;
char buffer[1024];
buffer[0] = 'f';
fprintf(stdout, "%s", buffer);
std::cout << "Hello" << std::endl;
int serverFd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
// bind & listen
struct sockaddr_in serverAddr;
memset(&serverAddr, 0, sizeof(serverAddr));
serverAddr.sin_family = AF_INET;
serverAddr.sin_addr.s_addr = htonl(INADDR_ANY);
serverAddr.sin_port = htons(80);
int bindResult = bind(serverFd, (struct sockaddr *) &serverAddr, sizeof(serverAddr));
if (bindResult < 0) {
fprintf(stderr, "Fail to bindn");
return 1;
}
int listenResult = listen(serverFd, 1024);
if (listenResult < 0) {
fprintf(stderr, "Fail to listenn");
return 1;
}
struct sockaddr clientAddr;
unsigned int clientlen = sizeof(clientAddr);
int acceptFd = accept(serverFd, &clientAddr, &clientlen);
if (acceptFd < 0) {
fprintf(stderr, "Fail to create client connection file descriptorn");
return 1;
}
int fd = acceptFd;
ssize_t received = recv(fd, &buffer, 1024, 0);
if (received < 0) {
fprintf(stderr, "Fail to received bytess from clientn");
if (errno == EINTR) {
fprintf(stderr, "Reason: EINTRn");
} else if (errno == EAGAIN || errno == EWOULDBLOCK) {
fprintf(stderr, "Reason: EAGAIN or EWOULDBLOCKn");
} else {
fprintf(stderr, "Reason: %dn", errno);
close(fd);
return 1;
}
} else if (received == 0) {
close(fd);
} else {
buffer[received] = '';
fprintf(stdout, "%s", buffer);
}
}

test_2.cpp(确实使用epoll()并且效果不佳)：

#include <iostream>
#include <sys/epoll.h>
#include <unistd.h>
#include <netinet/in.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <fcntl.h>
#include <signal.h>
#include <string.h>
int main() {
// TODO: too much error message to handle, so it's necessary to deal with it (maybe macros can)
std::cout << "Hello" << std::endl;
// process ignore SIGPIPE which is caused by send(), or process will exit, which is hard to find out
signal(SIGPIPE, SIG_IGN);
// here needs a socket fd or other fd
// well, AF_INET is okay;socket(PF_INET, SOCK_SEQPACKET, 0) is sctp, tcp cannot use SOCK_SEQPACKET :(
// when using tcp, watch out **record boundaries**
int serverFd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
if (serverFd < 0) {
fprintf(stderr, "Fail to create socket file descriptorn");
return 1;
}
// nonblock
// nonblock
int flags = fcntl(serverFd, F_GETFL, 0);
if (flags < 0) {
fprintf(stderr, "Fail to get flagsn");
return 1;
}
int setFlagResult = fcntl(serverFd, F_SETFL, flags | O_NONBLOCK);
if (setFlagResult < 0) {
fprintf(stderr, "Fail to set flagsn");
return 1;
}
// bind & listen
struct sockaddr_in serverAddr;
memset(&serverAddr, 0, sizeof(serverAddr));
serverAddr.sin_family = AF_INET;
serverAddr.sin_addr.s_addr = htonl(INADDR_ANY);
serverAddr.sin_port = htons(80);
int bindResult = bind(serverFd, (struct sockaddr *) &serverAddr, sizeof(serverAddr));
if (bindResult < 0) {
fprintf(stderr, "Fail to bindn");
return 1;
}
int listenResult = listen(serverFd, 1024);
if (listenResult < 0) {
fprintf(stderr, "Fail to listenn");
return 1;
}
// epoll fd
int epollFd = epoll_create(1);
if (epollFd < 0) {
fprintf(stderr, "Fail to create epoll file descriptorn");
return 1;
}
// event
struct epoll_event event, events[1024];
event.events = EPOLLIN;
event.data.fd = serverFd;
// ctl
int ctlResult = epoll_ctl(epollFd, EPOLL_CTL_ADD, serverFd, &event);
if (ctlResult < 0) {
fprintf(stderr, "Fail to run epoll_ctln");
return 1;
}
// wait
while (1) {
int event_count = epoll_wait(epollFd, events, 1024, -1);
for (int i = 0; i < event_count; i++) {
struct epoll_event event = events[i];
// accept
if (event.data.fd == serverFd) {
unsigned int clientlen = sizeof(clientAddr);
int acceptFd = accept(serverFd, (struct sockaddr *) &clientAddr, &clientlen);
if (acceptFd < 0) {
fprintf(stderr, "Fail to create client connection file descriptorn");
fprintf(stderr, "Fail Reason: %dn", errno);
return 1;
}
// nonblock
int flags = fcntl(acceptFd, F_GETFL, 0);
if (flags < 0) {
fprintf(stderr, "Fail to get flagsn");
return 1;
}
int setFlagResult = fcntl(acceptFd, F_SETFL, flags | O_NONBLOCK);
if (setFlagResult < 0) {
fprintf(stderr, "Fail to set flagsn");
return 1;
}
struct epoll_event event;
event.events = EPOLLIN;
event.data.fd = serverFd;
int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_ADD, acceptFd, &event);
if (ctlClientResult < 0) {
fprintf(stderr, "Fail to run epoll_ctln");
return 1;
}
// client recv
} else if (event.events & EPOLLIN) {
int fd = event.data.fd;
char buffer[1024+1];
ssize_t received = recv(fd, &buffer, 1024, 0);
if (received < 0) {
fprintf(stderr, "Fail to received bytess from clientn");
if (errno == EINTR) {
fprintf(stderr, "Reason: EINTRn");
} else if (errno == EAGAIN || errno == EWOULDBLOCK) {
fprintf(stderr, "Reason: EAGAIN or EWOULDBLOCKn");
} else {
fprintf(stderr, "Reason: %dn", errno);
close(fd);
int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_DEL, fd, &event);
if (ctlClientResult < 0) {
fprintf(stderr, "Fail to run epoll_ctln");
return 1;
}
return 1;
}
} else if (received == 0) {
int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_DEL, fd, &event);
if (ctlClientResult < 0) {
fprintf(stderr, "Fail to run epoll_ctln");
return 1;
}
close(fd);
} else {
buffer[received] = '';
fprintf(stdout, "%s", buffer);
// if you want to send something...
event.events |= EPOLLOUT;
// here is some data that event can hold
event.data.u32 = (uint32_t) 1;
// you can now send data or just put event in epoll, which is maybe easier
int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_MOD, fd, &event);
if (ctlClientResult < 0) {
fprintf(stderr, "Fail to run epoll_ctln");
return 1;
}
}
// client send
}  else if (event.events & EPOLLOUT) {
int fd = event.data.fd;
char buffer[] = "I see you";
ssize_t sendResult = send(fd, &buffer, 1024, 0);
if (sendResult < 0) {
fprintf(stderr, "Fail to received bytess from clientn");
if (errno == EINTR) {
fprintf(stderr, "Reason: EINTRn");
} else if (errno == EAGAIN || errno == EWOULDBLOCK) {
fprintf(stderr, "Reason: EAGAIN or EWOULDBLOCKn");
} else {
if (errno == EPIPE) {
fprintf(stderr, "Reason: EPIPEn");
} else {
fprintf(stderr, "Reason: %dn", errno);
}
close(fd);
int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_DEL, fd, &event);
if (ctlClientResult < 0) {
fprintf(stderr, "Fail to run epoll_ctln");
return 1;
}
return 1;
}
} else if (sendResult == 0) {
event.events = EPOLLIN;
int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_MOD, fd, &event);
if (ctlClientResult < 0) {
fprintf(stderr, "Fail to run epoll_ctln");
return 1;
}
} else {
// if you want to recv something...
//                    event.events |= EPOLLIN;
//                    // you can now send data or just put event in epoll, which is maybe easier
//                    int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_MOD, fd, &event);
//                    if (ctlClientResult < 0) {
//                        fprintf(stderr, "Fail to run epoll_ctln");
//                        return 1;
//                    }
}
}

}
}
return 0;
}

当我尝试建立TCP套接字连接(例如curl -v "http://host:80/"，它可以使test2.cpp在第81行运行)时，acceptFd< 0，并且根据第84行11errno，这意味着"避免了资源死锁"。

为什么？没有任何与线程相关的代码，是吗？

Answer 1

struct epoll_event event;
event.events = EPOLLIN;
event.data.fd = serverFd;
int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_ADD, acceptFd, &event);

当您将新接受的连接添加到 epoll 集时，您告诉它将其报告为serverFd上的命中。因此，当客户端向您发送数据时，您尝试接受新连接。

将event.data.fd = serverFd更改为event.data.fd = acceptFd。

然后你可以继续下一个错误：

char buffer[] = "I see you";
ssize_t sendResult = send(fd, &buffer, 1024, 0);

1024？！

此外，无论何时使用非阻塞套接字，都应添加代码以将EAGAIN或EWOULDBLOCK错误作为非致命错误进行处理。

Answer 2

错误代码 11 是EAGAIN，这是处理非阻塞套接字 I/O 时非常常见的错误代码。这意味着请求的操作无关，请稍后重试。

在accept()的情况下，这意味着：

EAGAIN或EWOULDBLOCK

套接字标记为非阻塞，并且不存在要接受的连接。POSIX.1-2001 和 POSIX.1-2008 允许在这种情况下返回任一错误，并且不要求这些常量具有相同的值，因此可移植应用程序应检查这两种可能性。

这意味着您在错误的时间调用accept()，当侦听套接字没有任何客户端可以接受时。仔细检查您的epoll()使用情况，它可能有一个逻辑错误，导致您过早地调用accept()。

例如，在accept()成功接受客户端后，您将在调用epoll_ctl(EPOLL_CTL_ADD)时将侦听套接字而不是客户端套接字分配给event.data.fd：

struct epoll_event event;
event.events = EPOLLIN;
event.data.fd = serverFd; // <-- HERE, SHOULD BE acceptFd INSTEAD!
int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_ADD, acceptFd, &event);

因此，当客户端有数据等待读取时，您的循环最终将在侦听套接字上调用accept()，而不是在客户端套接字上调用recv()。

此外，您没有检查报告的event.events字段中的套接字错误。 如果客户端套接字上发生错误，则报告的events可能包括EPOLLERR和/或EPOLLHUP标志。 您应该检查这些标志，如果存在，则关闭客户端套接字，然后再检查EPOLLIN标志以调用recv()。

另请注意，在您的示例中，在套接字上调用close()之前，无需调用epoll_ctl(EPOLL_CTL_DEL)。 关闭套接字文件描述符将自动将其从监视它的 epoll 实例中删除。