diff --git a/client/port.cpp b/client/port.cpp index 5ed914d..8d99cfd 100644 --- a/client/port.cpp +++ b/client/port.cpp @@ -230,7 +230,8 @@ bool Port::openStreams(QString fileName, bool append, QString &error) OstProto::StreamConfigList streams; //if (!fileFormat.openStreams(fileName, streams, error)) - if (!pdmlFileFormat.openStreams(fileName, streams, error)) + //if (!pdmlFileFormat.openStreams(fileName, streams, error)) + if (!pcapFileFormat.openStreams(fileName, streams, error)) goto _fail; if (!append) diff --git a/common/pcapfileformat.cpp b/common/pcapfileformat.cpp index 3ef8c76..24f1df9 100644 --- a/common/pcapfileformat.cpp +++ b/common/pcapfileformat.cpp @@ -1,5 +1,5 @@ /* -Copyright (C) 2010 Srivats P. +Copyright (C) 2011 Srivats P. This file is part of "Ostinato" @@ -18,11 +18,15 @@ along with this program. If not, see */ #include "pcapfileformat.h" + +#include "pdml_p.h" #include "streambase.h" #include "hexdump.pb.h" #include #include +#include +#include #include static inline quint32 swap32(quint32 val) @@ -39,28 +43,12 @@ static inline quint16 swap16(quint16 val) ((val << 8) && 0xFF00)); } -typedef struct { - quint32 magicNumber; /* magic number */ - quint16 versionMajor; /* major version number */ - quint16 versionMinor; /* minor version number */ - qint32 thisZone; /* GMT to local correction */ - quint32 sigfigs; /* accuracy of timestamps */ - quint32 snapLen; /* max length of captured packets, in octets */ - quint32 network; /* data link type */ -} PcapFileHeader; - const quint32 kPcapFileMagic = 0xa1b2c3d4; const quint32 kPcapFileMagicSwapped = 0xd4c3b2a1; const quint16 kPcapFileVersionMajor = 2; const quint16 kPcapFileVersionMinor = 4; const quint32 kMaxSnapLen = 65535; - -typedef struct { - quint32 tsSec; /* timestamp seconds */ - quint32 tsUsec; /* timestamp microseconds */ - quint32 inclLen; /* number of octets of packet saved in file */ - quint32 origLen; /* actual length of packet */ -} PcapPacketHeader; +const quint32 kDltEthernet = 1; PcapFileFormat pcapFileFormat; @@ -75,54 +63,140 @@ PcapFileFormat::~PcapFileFormat() bool PcapFileFormat::openStreams(const QString fileName, OstProto::StreamConfigList &streams, QString &error) { + bool viaPdml = true; // TODO: shd be a param to function + bool isOk = false; + int pktCount; QFile file(fileName); - QDataStream fd; + QTemporaryFile file2; quint32 magic; + uchar gzipMagic[2]; + int len; PcapFileHeader fileHdr; PcapPacketHeader pktHdr; - quint32 len; - int pktCount = 1; QByteArray pktBuf; if (!file.open(QIODevice::ReadOnly)) goto _err_open; - if (file.size() < sizeof(fileHdr)) - goto _err_truncated; + len = file.peek((char*)gzipMagic, sizeof(gzipMagic)); + if (len < int(sizeof(gzipMagic))) + goto _err_reading_magic; - fd.setDevice(&file); + if ((gzipMagic[0] == 0x1f) && (gzipMagic[1] == 0x8b)) + { + QProcess gzip; - fd >> magic; + if (!file2.open()) + { + error.append("Unable to open temporary file to uncompress .gz\n"); + goto _err_unzip_fail; + } + qDebug("decompressing to %s", file2.fileName().toAscii().constData()); + + gzip.setStandardOutputFile(file2.fileName()); + // FIXME: hardcoded prog name + gzip.start("C:/Program Files/CmdLineTools/gzip.exe", + QStringList() + << "-d" + << "-c" + << fileName); + if (!gzip.waitForStarted(-1)) + { + error.append(QString("Unable to start gzip\n")); + goto _err_unzip_fail; + } + + if (!gzip.waitForFinished(-1)) + { + error.append(QString("Error running gzip\n")); + goto _err_unzip_fail; + } + + file2.seek(0); + + fd_.setDevice(&file2); + } + else + { + fd_.setDevice(&file); + } + + fd_ >> magic; + + qDebug("magic = %08x", magic); if (magic == kPcapFileMagicSwapped) { // Toggle Byte order - if (fd.byteOrder() == QDataStream::BigEndian) - fd.setByteOrder(QDataStream::LittleEndian); + if (fd_.byteOrder() == QDataStream::BigEndian) + fd_.setByteOrder(QDataStream::LittleEndian); else - fd.setByteOrder(QDataStream::BigEndian); + fd_.setByteOrder(QDataStream::BigEndian); } else if (magic != kPcapFileMagic) goto _err_bad_magic; - fd >> fileHdr.versionMajor; - fd >> fileHdr.versionMinor; - fd >> fileHdr.thisZone; - fd >> fileHdr.sigfigs; - fd >> fileHdr.snapLen; - fd >> fileHdr.network; + fd_ >> fileHdr.versionMajor; + fd_ >> fileHdr.versionMinor; + fd_ >> fileHdr.thisZone; + fd_ >> fileHdr.sigfigs; + fd_ >> fileHdr.snapLen; + fd_ >> fileHdr.network; if ((fileHdr.versionMajor != kPcapFileVersionMajor) || (fileHdr.versionMinor != kPcapFileVersionMinor)) goto _err_unsupported_version; - // TODO: what do we do about non-ethernet networks? +#if 1 + // XXX: we support only Ethernet, for now + if (fileHdr.network != kDltEthernet) + goto _err_unsupported_encap; +#endif pktBuf.resize(fileHdr.snapLen); - while (!fd.atEnd()) + if (viaPdml) + { + QTemporaryFile pdmlFile; + PdmlReader reader(&streams); + QProcess tshark; + + if (!pdmlFile.open()) + { + error.append("Unable to open temporary file to create PDML\n"); + goto _non_pdml; + } + + qDebug("generating PDML %s", pdmlFile.fileName().toAscii().constData()); + + tshark.setStandardOutputFile(pdmlFile.fileName()); + // FIXME: hardcoded prog name + tshark.start("C:/Program Files/Wireshark/Tshark.exe", + QStringList() + << QString("-r%1").arg(fileName) + << "-Tpdml"); + if (!tshark.waitForStarted(-1)) + { + error.append(QString("Unable to start tshark\n")); + goto _non_pdml; + } + + if (!tshark.waitForFinished(-1)) + { + error.append(QString("Error running tshark\n")); + goto _non_pdml; + } + + isOk = reader.read(&pdmlFile, this); // TODO: pass error string? + goto _exit; + + } + +_non_pdml: + pktCount = 1; + while (!fd_.atEnd()) { OstProto::Stream *stream = streams.add_stream(); OstProto::Protocol *proto = stream->add_protocol(); @@ -133,21 +207,11 @@ bool PcapFileFormat::openStreams(const QString fileName, proto->mutable_protocol_id()->set_id( OstProto::Protocol::kHexDumpFieldNumber); - // read PcapPacketHeader - fd >> pktHdr.tsSec; - fd >> pktHdr.tsUsec; - fd >> pktHdr.inclLen; - fd >> pktHdr.origLen; - - // TODO: chk fd.status() + readPacket(pktHdr, pktBuf); // validations on inclLen <= origLen && inclLen <= snapLen Q_ASSERT(pktHdr.inclLen <= fileHdr.snapLen); // TODO: convert to if - // read Pkt contents - len = fd.readRawData(pktBuf.data(), pktHdr.inclLen); // TODO: use while? - Q_ASSERT(len == pktHdr.inclLen); // TODO: remove assert - hexDump->set_content(pktBuf.data(), pktHdr.inclLen); hexDump->set_pad_until_end(false); @@ -159,6 +223,14 @@ bool PcapFileFormat::openStreams(const QString fileName, isOk = true; goto _exit; +#if 1 +_err_unsupported_encap: + error = QString(tr("%1 has non-ethernet encapsulation (%2) which is " + "not supported - Sorry!")) + .arg(fileName).arg(fileHdr.network); + goto _exit; +#endif + _err_unsupported_version: error = QString(tr("%1 is in PCAP version %2.%3 format which is " "not supported - Sorry!")) @@ -169,24 +241,65 @@ _err_bad_magic: error = QString(tr("%1 is not a valid PCAP file")).arg(fileName); goto _exit; +#if 0 _err_truncated: error = QString(tr("%1 is too short")).arg(fileName); goto _exit; +#endif + +_err_unzip_fail: + goto _exit; + +_err_reading_magic: + error = QString(tr("Unable to read magic from %1")).arg(fileName); + goto _exit; _err_open: error = QString(tr("Unable to open file: %1")).arg(fileName); goto _exit; _exit: + file.close(); return isOk; } +/*! + Reads packet meta data into pktHdr and packet content into buf. + + Returns true if packet is read successfully, false otherwise. +*/ +bool PcapFileFormat::readPacket(PcapPacketHeader &pktHdr, QByteArray &pktBuf) +{ + quint32 len; + + // TODO: chk fd_.status() + + // read PcapPacketHeader + fd_ >> pktHdr.tsSec; + fd_ >> pktHdr.tsUsec; + fd_ >> pktHdr.inclLen; + fd_ >> pktHdr.origLen; + + // TODO: chk fd_.status() + + // XXX: should never be required, but we play safe + if (quint32(pktBuf.size()) < pktHdr.inclLen) + pktBuf.resize(pktHdr.inclLen); + + // read Pkt contents + len = fd_.readRawData(pktBuf.data(), pktHdr.inclLen); // TODO: use while? + + Q_ASSERT(len == pktHdr.inclLen); // TODO: remove assert + pktBuf.resize(len); + + return true; +} + bool PcapFileFormat::saveStreams(const OstProto::StreamConfigList streams, const QString fileName, QString &error) { bool isOk = false; QFile file(fileName); - QDataStream fd; PcapFileHeader fileHdr; PcapPacketHeader pktHdr; QByteArray pktBuf; @@ -194,7 +307,7 @@ bool PcapFileFormat::saveStreams(const OstProto::StreamConfigList streams, if (!file.open(QIODevice::WriteOnly)) goto _err_open; - fd.setDevice(&file); + fd_.setDevice(&file); fileHdr.magicNumber = kPcapFileMagic; fileHdr.versionMajor = kPcapFileVersionMajor; @@ -202,15 +315,15 @@ bool PcapFileFormat::saveStreams(const OstProto::StreamConfigList streams, fileHdr.thisZone = 0; fileHdr.sigfigs = 0; fileHdr.snapLen = kMaxSnapLen; - fileHdr.network = 1; // Ethernet; FIXME: Hardcoding + fileHdr.network = kDltEthernet; - fd << fileHdr.magicNumber; - fd << fileHdr.versionMajor; - fd << fileHdr.versionMinor; - fd << fileHdr.thisZone; - fd << fileHdr.sigfigs; - fd << fileHdr.snapLen; - fd << fileHdr.network; + fd_ << fileHdr.magicNumber; + fd_ << fileHdr.versionMajor; + fd_ << fileHdr.versionMinor; + fd_ << fileHdr.thisZone; + fd_ << fileHdr.sigfigs; + fd_ << fileHdr.snapLen; + fd_ << fileHdr.network; pktBuf.resize(kMaxSnapLen); @@ -230,11 +343,11 @@ bool PcapFileFormat::saveStreams(const OstProto::StreamConfigList streams, if (pktHdr.inclLen > fileHdr.snapLen) pktHdr.inclLen = fileHdr.snapLen; - fd << pktHdr.tsSec; - fd << pktHdr.tsUsec; - fd << pktHdr.inclLen; - fd << pktHdr.origLen; - fd.writeRawData(pktBuf.data(), pktHdr.inclLen); + fd_ << pktHdr.tsSec; + fd_ << pktHdr.tsUsec; + fd_ << pktHdr.inclLen; + fd_ << pktHdr.origLen; + fd_.writeRawData(pktBuf.data(), pktHdr.inclLen); } file.close(); diff --git a/common/pcapfileformat.h b/common/pcapfileformat.h index a515e32..04a02ec 100644 --- a/common/pcapfileformat.h +++ b/common/pcapfileformat.h @@ -1,5 +1,5 @@ /* -Copyright (C) 2010 Srivats P. +Copyright (C) 2011 Srivats P. This file is part of "Ostinato" @@ -21,12 +21,16 @@ along with this program. If not, see #include "protocol.pb.h" +#include #include +class PdmlReader; class PcapFileFormat : public QObject { Q_OBJECT + friend class PdmlReader; + public: PcapFileFormat(); ~PcapFileFormat(); @@ -35,6 +39,28 @@ public: OstProto::StreamConfigList &streams, QString &error); bool saveStreams(const OstProto::StreamConfigList streams, const QString fileName, QString &error); + +private: + typedef struct { + quint32 magicNumber; /* magic number */ + quint16 versionMajor; /* major version number */ + quint16 versionMinor; /* minor version number */ + qint32 thisZone; /* GMT to local correction */ + quint32 sigfigs; /* accuracy of timestamps */ + quint32 snapLen; /* max length of captured packets, in octets */ + quint32 network; /* data link type */ + } PcapFileHeader; + + typedef struct { + quint32 tsSec; /* timestamp seconds */ + quint32 tsUsec; /* timestamp microseconds */ + quint32 inclLen; /* number of octets of packet saved in file */ + quint32 origLen; /* actual length of packet */ + } PcapPacketHeader; + + bool readPacket(PcapPacketHeader &pktHdr, QByteArray &pktBuf); + + QDataStream fd_; }; extern PcapFileFormat pcapFileFormat; diff --git a/common/pdml_p.cpp b/common/pdml_p.cpp index 37e7aa2..d681667 100644 --- a/common/pdml_p.cpp +++ b/common/pdml_p.cpp @@ -19,8 +19,10 @@ along with this program. If not, see #include "pdml_p.h" +#include "PcapFileFormat.h" #include "mac.pb.h" #include "eth2.pb.h" +#include "dot3.pb.h" #include "hexdump.pb.h" #include "ip4.pb.h" #include "ip6.pb.h" @@ -349,6 +351,7 @@ bool PdmlParser::fatalError(const QXmlParseException &exception) PdmlReader::PdmlReader(OstProto::StreamConfigList *streams) { gPdmlReader = this; + pcap_ = NULL; pass_ = 0; streams_ = streams; @@ -358,20 +361,21 @@ PdmlReader::PdmlReader(OstProto::StreamConfigList *streams) #if 0 factory_.insert("fake-field-wrapper", new PdmlFakeFieldWrapperProtocol()); +#endif factory_.insert("eth",PdmlEthProtocol::createInstance); factory_.insert("ip",PdmlIp4Protocol::createInstance); factory_.insert("ipv6",PdmlIp6Protocol::createInstance); factory_.insert("tcp",PdmlTcpProtocol::createInstance); -#endif } PdmlReader::~PdmlReader() { } -bool PdmlReader::read(QIODevice *device) +bool PdmlReader::read(QIODevice *device, PcapFileFormat *pcap) { setDevice(device); + pcap_ = pcap; packetCount_ = 0; // 1st pass - preprocessing fake fields @@ -408,7 +412,14 @@ bool PdmlReader::read(QIODevice *device) } } - return !error(); + if (error()) + { + qDebug("Line %lld", lineNumber()); + qDebug("Col %lld", columnNumber()); + qDebug("%s", errorString().toAscii().constData()); // FIXME + return false; + } + return true; } // TODO: use a temp pool to avoid a lot of new/delete @@ -502,6 +513,8 @@ void PdmlReader::readPdml() } } +/////////////////////// PASS 1 ////////////////////////// + void PdmlReader::readPacketPass1() { Q_ASSERT(isStartElement() && name() == "packet"); @@ -614,17 +627,24 @@ void PdmlReader::readFieldPass1() } } +/////////////////////// PASS 2 ////////////////////////// + void PdmlReader::readPacket() { - //Q_ASSERT(isStartElement() && name() == "packet"); + PcapFileFormat::PcapPacketHeader pktHdr; + + Q_ASSERT(isStartElement() && name() == "packet"); qDebug("%s: packetNum = %d", __FUNCTION__, packetCount_); - // XXX: For now, each packet is converted to a stream + // XXX: we play dumb and convert each packet to a stream, for now currentStream_ = streams_->add_stream(); currentStream_->mutable_stream_id()->set_id(packetCount_); currentStream_->mutable_core()->set_is_enabled(true); + if (pcap_) + pcap_->readPacket(pktHdr, pktBuf_); + while (!atEnd()) { readNext(); @@ -668,9 +688,23 @@ void PdmlReader::readProto() Q_ASSERT(isStartElement() && name() == "proto"); QString protoName = attributes().value("name").toString(); - qDebug("proto: %s", protoName.toAscii().constData()); + int pos = -1; -#if 0 + if (!attributes().value("pos").isEmpty()) + pos = attributes().value("pos").toString().toInt(); + + qDebug("proto: %s, pos = %d", protoName.toAscii().constData(), pos); + + // This is a heuristic to skip protocols which are not part of + // this frame, but of a reassembled segment spanning several frames + if ((pos == 0) && (currentStream_->protocol_size() > 0)) + { + qDebug("(skipped)"); + skipElement(); + return; + } + +#if 1 if (protoName.isEmpty() || (protoName == "expert")) { skipElement(); @@ -678,6 +712,34 @@ void PdmlReader::readProto() } #endif + if (!factory_.contains(protoName) && pcap_) + { + int pos = -1; + int size = -1; + + if (!attributes().value("pos").isEmpty()) + pos = attributes().value("pos").toString().toInt(); + if (!attributes().value("size").isEmpty()) + size = attributes().value("size").toString().toInt(); + + if (pos >= 0 && size > 0 + && ((pos + size) <= pktBuf_.size())) + { + OstProto::Protocol *proto = currentStream_->add_protocol(); + OstProto::HexDump *hexDump = proto->MutableExtension( + OstProto::hexDump); + + proto->mutable_protocol_id()->set_id( + OstProto::Protocol::kHexDumpFieldNumber); + + hexDump->set_content(pktBuf_.constData() + pos, size); + hexDump->set_pad_until_end(false); + + skipElement(); + return; + } + } + pdmlProto = allocPdmlProtocol(protoName); Q_ASSERT(pdmlProto != NULL); @@ -902,6 +964,11 @@ void PdmlUnknownProtocol::preProtocolHandler(QString name, if (!isOk) goto _skip_pos_size_proc; + // If pos+size goes beyond the frame length, this is a "reassembled" + // protocol and should be skipped + if (quint32(pos + size) > stream->core().frame_len()) + goto _skip_pos_size_proc; + expPos_ = pos; endPos_ = expPos_ + size; @@ -1048,6 +1115,8 @@ void PdmlFakeFieldWrapperProtocol::postProtocolHandler(OstProto::Stream *stream) qDebug("%s: expPos_ = %d\n", __FUNCTION__, expPos_); + // TODO: if content size is zero, remove protocol? + hexDump->set_pad_until_end(false); expPos_ = -1; } @@ -1059,6 +1128,7 @@ void PdmlFakeFieldWrapperProtocol::unknownFieldHandler(QString name, int pos, stream->protocol_size()-1)->MutableExtension(OstProto::hexDump); if ((pos == expPos_) && (size >= 0) && + (!name.startsWith("tcp.segment")) && (!attributes.value("unmaskedvalue").isEmpty() || !attributes.value("value").isEmpty())) { @@ -1106,6 +1176,36 @@ void PdmlEthProtocol::unknownFieldHandler(QString name, int pos, int size, eth2->set_type(attributes.value("value").toString().toUInt(&isOk, kBaseHex)); eth2->set_is_override_type(true); } + else if (name == "eth.len") + { + OstProto::Protocol *proto = stream->add_protocol(); + + proto->mutable_protocol_id()->set_id( + OstProto::Protocol::kDot3FieldNumber); + + OstProto::Dot3 *dot3 = proto->MutableExtension(OstProto::dot3); + + bool isOk; + dot3->set_length(attributes.value("value").toString().toUInt(&isOk, kBaseHex)); + dot3->set_is_override_length(true); + } + else if (name == "eth.trailer") + { + QByteArray trailer = QByteArray::fromHex( + attributes.value("value").toString().toUtf8()); + + stream->mutable_core()->mutable_name()->append(trailer.constData(), + trailer.size()); + } + else if ((name == "eth.fcs") || + attributes.value("show").toString().startsWith("Frame check sequence")) + { + QByteArray trailer = QByteArray::fromHex( + attributes.value("value").toString().toUtf8()); + + stream->mutable_core()->mutable_name()->append(trailer.constData(), + trailer.size()); + } } @@ -1141,7 +1241,13 @@ void PdmlIp4Protocol::unknownFieldHandler(QString name, int pos, int size, { bool isOk; - if (name == "ip.flags") + if ((name == "ip.options") || + attributes.value("show").toString().startsWith("Options")) + { + options_ = QByteArray::fromHex( + attributes.value("value").toString().toUtf8()); + } + else if (name == "ip.flags") { OstProto::Ip4 *ip4 = stream->mutable_protocol( stream->protocol_size()-1)->MutableExtension(OstProto::ip4); @@ -1160,6 +1266,21 @@ void PdmlIp4Protocol::postProtocolHandler(OstProto::Stream *stream) ip4->set_is_override_totlen(true); // FIXME ip4->set_is_override_proto(true); // FIXME ip4->set_is_override_cksum(true); // FIXME + + if (options_.size()) + { + OstProto::Protocol *proto = stream->add_protocol(); + + proto->mutable_protocol_id()->set_id( + OstProto::Protocol::kHexDumpFieldNumber); + + OstProto::HexDump *hexDump = proto->MutableExtension(OstProto::hexDump); + + hexDump->mutable_content()->append(options_.constData(), + options_.size()); + hexDump->set_pad_until_end(false); + options_.resize(0); + } } // ---------------------------------------------------------- // @@ -1255,8 +1376,8 @@ void PdmlTcpProtocol::unknownFieldHandler(QString name, int pos, int size, && attributes.value("show").toString().startsWith("TCP segment data")) { segmentData_ = QByteArray::fromHex(attributes.value("value").toString().toUtf8()); - stream->mutable_core()->mutable_name()->append(segmentData_.constData(), - segmentData_.size()); + stream->mutable_core()->mutable_name()->insert(0, + segmentData_.constData(), segmentData_.size()); } } diff --git a/common/pdml_p.h b/common/pdml_p.h index bc07f16..242d3eb 100644 --- a/common/pdml_p.h +++ b/common/pdml_p.h @@ -91,6 +91,7 @@ private: #endif class PdmlUnknownProtocol; +class PcapFileFormat; class PdmlReader : public QXmlStreamReader { friend class PdmlUnknownProtocol; @@ -98,7 +99,7 @@ public: PdmlReader(OstProto::StreamConfigList *streams); ~PdmlReader(); - bool read(QIODevice *device); + bool read(QIODevice *device, PcapFileFormat *pcap = NULL); private: PdmlDefaultProtocol* allocPdmlProtocol(QString protoName); @@ -144,12 +145,15 @@ private: QMap factory_; OstProto::StreamConfigList *streams_; + PcapFileFormat *pcap_; int pass_; int packetCount_; OstProto::Stream *currentStream_; QList pktFragments_; + QByteArray pktBuf_; + //PdmlDefaultProtocol *currentPdmlProtocol_; //google::protobuf::Message *currentProtocolMsg_; }; @@ -230,6 +234,8 @@ public: virtual void unknownFieldHandler(QString name, int pos, int size, const QXmlStreamAttributes &attributes, OstProto::Stream *stream); virtual void postProtocolHandler(OstProto::Stream *stream); +private: + QByteArray options_; }; class PdmlIp6Protocol : public PdmlDefaultProtocol diff --git a/test/main.cpp b/test/main.cpp index 0b98cfd..c69ccbd 100644 --- a/test/main.cpp +++ b/test/main.cpp @@ -26,15 +26,17 @@ int main(int argc, char* argv[]) QString inFile(argv[1]); QString outFile(argv[2]); - if (!pdmlFileFormat.openStreams(inFile, streams, error)) + if (!pcapFileFormat.openStreams(inFile, streams, error)) { - fprintf(stdout, "failed reading streams from %s\n", inFile.toAscii().constData()); + fprintf(stdout, "failed reading streams from %s:%s\n", + inFile.toAscii().constData(), error.toAscii().constData()); exit(1); } if (!pcapFileFormat.saveStreams(streams, outFile, error)) { - fprintf(stdout, "failed writing streams to %s\n", outFile.toAscii().constData()); + fprintf(stdout, "failed writing streams to %s:%s\n", + outFile.toAscii().constData(), error.toAscii().constData()); exit(1); }