From 9cac3cd91857be34e85687906fe7f40157c10d51 Mon Sep 17 00:00:00 2001 From: "Srivats P." Date: Sat, 10 Mar 2012 13:29:41 +0530 Subject: [PATCH] Linux now uses stats using netlink - if that isn't available for some reason, we fallback to the /proc stats. If netlink stats is used, link state info is available. --- server/linuxport.cpp | 436 +++++++++++++++++++++++++++++++++++++++---- server/linuxport.h | 9 + 2 files changed, 413 insertions(+), 32 deletions(-) diff --git a/server/linuxport.cpp b/server/linuxport.cpp index 0f041f1..74e7c46 100644 --- a/server/linuxport.cpp +++ b/server/linuxport.cpp @@ -22,6 +22,8 @@ along with this program. If not, see #ifdef Q_OS_LINUX #include +#include +#include #include #include @@ -31,12 +33,15 @@ along with this program. If not, see #include #include +#include + QList LinuxPort::allPorts_; LinuxPort::StatsMonitor *LinuxPort::monitor_; LinuxPort::LinuxPort(int id, const char *device) : PcapPort(id, device) { + isPromisc_ = true; clearPromisc_ = false; // We don't need per port Rx/Tx monitors for Linux @@ -93,15 +98,17 @@ LinuxPort::~LinuxPort() void LinuxPort::init() { - // TODO: Update Notes with Promisc/Non-Promisc - if (!monitor_->isRunning()) monitor_->start(); + + monitor_->waitForSetupFinished(); + + if (!isPromisc_) + addNote("Non Promiscuous Mode"); } OstProto::LinkState LinuxPort::linkState() { - // TODO return linkState_; } @@ -121,12 +128,29 @@ LinuxPort::StatsMonitor::StatsMonitor() : QThread() { stop_ = false; + setupDone_ = false; + ioctlSocket_ = socket(AF_INET, SOCK_DGRAM, 0); + Q_ASSERT(ioctlSocket_ >= 0); +} + +LinuxPort::StatsMonitor::~StatsMonitor() +{ + close(ioctlSocket_); } void LinuxPort::StatsMonitor::run() +{ + if (netlinkStats() < 0) + { + qDebug("netlink stats not available - using /proc stats"); + procStats(); + } +} + +void LinuxPort::StatsMonitor::procStats() { PortStats **portStats; - int fd, sd; + int fd; QByteArray buf; int len; char *p, *end; @@ -136,7 +160,6 @@ void LinuxPort::StatsMonitor::run() "%llu%llu%llu%llu%llu%llu%n%n%llu%llu%u%u%u%u%u%n\n", }; const char *fmt; - struct ifreq ifr; // // We first setup stuff before we start polling for stats @@ -184,10 +207,6 @@ void LinuxPort::StatsMonitor::run() portStats = (PortStats**) calloc(count, sizeof(PortStats)); Q_ASSERT(portStats != NULL); - sd = socket(AF_INET, SOCK_DGRAM, 0); - Q_ASSERT(sd >= 0); - memset(&ifr, 0, sizeof(ifr)); - // // Populate the port stats array // @@ -224,28 +243,11 @@ void LinuxPort::StatsMonitor::run() { portStats[index] = &(port->stats_); - // Set promisc mode, if not already set - strncpy(ifr.ifr_name, port->name(), sizeof(ifr.ifr_name)); - if (ioctl(sd, SIOCGIFFLAGS, &ifr) != -1) - { - if ((ifr.ifr_flags & IFF_PROMISC) == 0) - { - ifr.ifr_flags |= IFF_PROMISC; - if (ioctl(sd, SIOCSIFFLAGS, &ifr) != -1) - port->clearPromisc_ = true; - else - { - qDebug("%s: failed to set promisc; " - "SIOCSIFFLAGS failed (%s)", - port->name(), strerror(errno)); - } - } - } + if (setPromisc(port->name())) + port->clearPromisc_ = true; else - { - qDebug("%s: failed to set promisc; SIOCGIFFLAGS failed (%s)", - port->name(), strerror(errno)); - } + port->isPromisc_ = false; + break; } } @@ -260,9 +262,8 @@ void LinuxPort::StatsMonitor::run() } Q_ASSERT(index == count); - close(sd); - qDebug("stats for %d ports setup", count); + setupDone_ = true; // // We are all set - Let's start polling for stats! @@ -353,8 +354,379 @@ void LinuxPort::StatsMonitor::run() free(portStats); } +int LinuxPort::StatsMonitor::netlinkStats() +{ + QHash portStats; + QHash linkState; + int fd; + struct sockaddr_nl local; + struct sockaddr_nl kernel; + QByteArray buf; + int len, count; + struct { + struct nlmsghdr nlh; + struct rtgenmsg rtg; + } ifListReq; + struct iovec iov; + struct msghdr msg; + struct nlmsghdr *nlm; + bool done = false; + + // + // We first setup stuff before we start polling for stats + // + fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE); + if (fd < 0) + { + qWarning("Unable to open netlink socket (errno %d)", errno); + return -1; + } + + memset(&local, 0, sizeof(local)); + local.nl_family = AF_NETLINK; + + if (bind(fd, (struct sockaddr*) &local, sizeof(local)) < 0) + { + qWarning("Unable to bind netlink socket (errno %d)", errno); + return -1; + } + + memset(&ifListReq, 0, sizeof(ifListReq)); + ifListReq.nlh.nlmsg_len = sizeof(ifListReq); + ifListReq.nlh.nlmsg_type = RTM_GETLINK; + ifListReq.nlh.nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP; + ifListReq.nlh.nlmsg_pid = 0; + ifListReq.rtg.rtgen_family = AF_PACKET; + + buf.fill('\0', 1024); + + msg.msg_name = &kernel; + msg.msg_namelen = sizeof(kernel); + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + msg.msg_control = NULL; + msg.msg_controllen = 0; + msg.msg_flags = 0; + + qDebug("nlmsg_flags = %x", ifListReq.nlh.nlmsg_flags); + + if (send(fd, (void*)&ifListReq, sizeof(ifListReq), 0) < 0) + { + qWarning("Unable to send GETLINK request (errno %d)", errno); + return -1; + } + + // Find required size of buffer and resize accordingly + while (1) + { + iov.iov_base = buf.data(); + iov.iov_len = buf.size(); + msg.msg_flags = 0; + + // Peek at reply to check buffer size required + len = recvmsg(fd, &msg, MSG_PEEK|MSG_TRUNC); + + if (len < 0) + { + if (errno == EINTR || errno == EAGAIN) + continue; + + qWarning("netlink recv error %d", errno); + return -1; + } + else if (len == 0) + { + qWarning("netlink closed the socket on my face!"); + return -1; + } + else + { + if (msg.msg_flags & MSG_TRUNC) + { + if (len == buf.size()) // Older Kernel returns truncated size + { + qDebug("netlink buffer size %d not enough", buf.size()); + qDebug("retrying with double the size"); + // Double the size and retry + buf.resize(buf.size()*2); + continue; + } + else // Newer Kernel returns actual size required + { + qDebug("netlink required buffer size = %d", len); + buf.resize(len); + continue; + } + } + else + qDebug("buffer size %d enough for netlink", buf.size()); + + break; + } + } + + count = 0; + +_retry: + msg.msg_flags = 0; + + // Actually receive the reply now + len = recvmsg(fd, &msg, 0); + + if (len < 0) + { + if (errno == EINTR || errno == EAGAIN) + goto _retry; + qWarning("netlink recv error %d", errno); + return -1; + } + else if (len == 0) + { + qWarning("netlink socket closed unexpectedly"); + return -1; + } + + // + // Populate the port stats hash table + // + nlm = (struct nlmsghdr*) buf.data(); + while (NLMSG_OK(nlm, (uint)len)) + { + struct ifinfomsg *ifi; + struct rtattr *rta; + int rtaLen; + char ifname[64] = ""; + + if (nlm->nlmsg_type == NLMSG_DONE) + { + done = true; + break; + } + + if (nlm->nlmsg_type == NLMSG_ERROR) + { + struct nlmsgerr *err = (struct nlmsgerr*) NLMSG_DATA(nlm); + qDebug("RTNETLINK error %d", err->error); + done = true; + break; + } + + Q_ASSERT(nlm->nlmsg_type == RTM_NEWLINK); + + ifi = (struct ifinfomsg*) NLMSG_DATA(nlm); + rta = IFLA_RTA(ifi); + rtaLen = len - NLMSG_LENGTH(sizeof(*ifi)); + while (RTA_OK(rta, rtaLen)) + { + if (rta->rta_type == IFLA_IFNAME) + { + strncpy(ifname, (char*)RTA_DATA(rta), RTA_PAYLOAD(rta)); + ifname[RTA_PAYLOAD(rta)] = 0; + break; + } + rta = RTA_NEXT(rta, rtaLen); + } + + qDebug("if: %s(%d)", ifname, ifi->ifi_index); + foreach(LinuxPort* port, allPorts_) + { + if (strcmp(port->name(), ifname) == 0) + { + portStats[uint(ifi->ifi_index)] = &(port->stats_); + linkState[uint(ifi->ifi_index)] = &(port->linkState_); + + if (setPromisc(port->name())) + port->clearPromisc_ = true; + else + port->isPromisc_ = false; + + count++; + break; + } + } + nlm = NLMSG_NEXT(nlm, len); + } + + if (!done) + goto _retry; + + qDebug("port count = %d\n", count); + if (count <= 0) + { + qWarning("no ports in RTNETLINK GET_LINK - no stats will be available"); + return - 1; + } + + qDebug("stats for %d ports setup", count); + setupDone_ = true; + + // + // We are all set - Let's start polling for stats! + // + while (!stop_) + { + if (send(fd, (void*)&ifListReq, sizeof(ifListReq), 0) < 0) + { + qWarning("Unable to send GETLINK request (errno %d)", errno); + goto _try_later; + } + + done = false; + +_retry_recv: + msg.msg_flags = 0; + len = recvmsg(fd, &msg, 0); + + if (len < 0) + { + if (errno == EINTR || errno == EAGAIN) + goto _retry_recv; + qWarning("netlink recv error %d", errno); + break; + } + else if (len == 0) + { + qWarning("netlink socket closed unexpectedly"); + break; + } + + nlm = (struct nlmsghdr*) buf.data(); + while (NLMSG_OK(nlm, (uint)len)) + { + struct ifinfomsg *ifi; + struct rtattr *rta; + int rtaLen; + + if (nlm->nlmsg_type == NLMSG_DONE) + { + done = true; + break; + } + + if (nlm->nlmsg_type == NLMSG_ERROR) + { + struct nlmsgerr *err = (struct nlmsgerr*) NLMSG_DATA(nlm); + qDebug("RTNETLINK error: %s", strerror(-err->error)); + done = true; + break; + } + + Q_ASSERT(nlm->nlmsg_type == RTM_NEWLINK); + + ifi = (struct ifinfomsg*) NLMSG_DATA(nlm); + rta = IFLA_RTA(ifi); + rtaLen = len - NLMSG_LENGTH(sizeof(*ifi)); + while (RTA_OK(rta, rtaLen)) + { + // TODO: IFLA_STATS64 + if (rta->rta_type == IFLA_STATS) + { + struct rtnl_link_stats *rtnlStats = + (struct rtnl_link_stats*) RTA_DATA(rta); + AbstractPort::PortStats *stats = portStats[ifi->ifi_index]; + OstProto::LinkState *state = linkState[ifi->ifi_index]; + + if (!stats) + break; + + stats->rxPps = (rtnlStats->rx_packets - stats->rxPkts) + / kRefreshFreq_; + stats->rxBps = (rtnlStats->rx_bytes - stats->rxBytes) + / kRefreshFreq_; + stats->rxPkts = rtnlStats->rx_packets; + stats->rxBytes = rtnlStats->rx_bytes; + stats->txPps = (rtnlStats->tx_packets - stats->txPkts) + / kRefreshFreq_; + stats->txBps = (rtnlStats->tx_bytes - stats->txBytes) + / kRefreshFreq_; + stats->txPkts = rtnlStats->tx_packets; + stats->txBytes = rtnlStats->tx_bytes; + + // TODO: export detailed error stats + stats->rxDrops = rtnlStats->rx_dropped + + rtnlStats->rx_missed_errors; + stats->rxErrors = rtnlStats->rx_errors; + stats->rxFifoErrors = rtnlStats->rx_fifo_errors; + stats->rxFrameErrors = rtnlStats->rx_crc_errors + + rtnlStats->rx_length_errors + + rtnlStats->rx_over_errors + + rtnlStats->rx_frame_errors; + + Q_ASSERT(state); + *state = ifi->ifi_flags & IFF_RUNNING ? + OstProto::LinkStateUp : OstProto::LinkStateDown; + + break; + } + rta = RTA_NEXT(rta, rtaLen); + } + nlm = NLMSG_NEXT(nlm, len); + } + + if (!done) + goto _retry_recv; + +_try_later: + QThread::sleep(kRefreshFreq_); + } + + portStats.clear(); + linkState.clear(); + + return 0; +} + +int LinuxPort::StatsMonitor::setPromisc(const char * portName) +{ + struct ifreq ifr; + + memset(&ifr, 0, sizeof(ifr)); + strncpy(ifr.ifr_name, portName, sizeof(ifr.ifr_name)); + + if (ioctl(ioctlSocket_, SIOCGIFFLAGS, &ifr) != -1) + { + if ((ifr.ifr_flags & IFF_PROMISC) == 0) + { + ifr.ifr_flags |= IFF_PROMISC; + if (ioctl(ioctlSocket_, SIOCSIFFLAGS, &ifr) != -1) + { + return 1; + } + else + { + qDebug("%s: failed to set promisc; " + "SIOCSIFFLAGS failed (%s)", + portName, strerror(errno)); + } + } + } + else + { + qDebug("%s: failed to set promisc; SIOCGIFFLAGS failed (%s)", + portName, strerror(errno)); + } + + return 0; +} + void LinuxPort::StatsMonitor::stop() { stop_ = true; } + +bool LinuxPort::StatsMonitor::waitForSetupFinished(int msecs) +{ + QTime t; + + t.start(); + while (!setupDone_) + { + if (t.elapsed() > msecs) + return false; + + QThread::msleep(10); + } + + return true; +} #endif diff --git a/server/linuxport.h b/server/linuxport.h index 817ece1..2658560 100644 --- a/server/linuxport.h +++ b/server/linuxport.h @@ -43,13 +43,22 @@ protected: { public: StatsMonitor(); + ~StatsMonitor(); void run(); void stop(); + bool waitForSetupFinished(int msecs = 10000); private: + int netlinkStats(); + void procStats(); + int setPromisc(const char* portName); + static const int kRefreshFreq_ = 1; // in seconds bool stop_; + bool setupDone_; + int ioctlSocket_; }; + bool isPromisc_; bool clearPromisc_; static QList allPorts_; static StatsMonitor *monitor_; // rx/tx stats for ALL ports