Snapshot before next improvement changes

This commit is contained in:
Srivats P. 2011-03-06 18:41:08 +05:30
parent b7ce0a6faf
commit 5af591b96b
6 changed files with 347 additions and 78 deletions

View File

@ -230,7 +230,8 @@ bool Port::openStreams(QString fileName, bool append, QString &error)
OstProto::StreamConfigList streams;
//if (!fileFormat.openStreams(fileName, streams, error))
if (!pdmlFileFormat.openStreams(fileName, streams, error))
//if (!pdmlFileFormat.openStreams(fileName, streams, error))
if (!pcapFileFormat.openStreams(fileName, streams, error))
goto _fail;
if (!append)

View File

@ -1,5 +1,5 @@
/*
Copyright (C) 2010 Srivats P.
Copyright (C) 2011 Srivats P.
This file is part of "Ostinato"
@ -18,11 +18,15 @@ along with this program. If not, see <http://www.gnu.org/licenses/>
*/
#include "pcapfileformat.h"
#include "pdml_p.h"
#include "streambase.h"
#include "hexdump.pb.h"
#include <QDataStream>
#include <QFile>
#include <QProcess>
#include <QTemporaryFile>
#include <QtGlobal>
static inline quint32 swap32(quint32 val)
@ -39,28 +43,12 @@ static inline quint16 swap16(quint16 val)
((val << 8) && 0xFF00));
}
typedef struct {
quint32 magicNumber; /* magic number */
quint16 versionMajor; /* major version number */
quint16 versionMinor; /* minor version number */
qint32 thisZone; /* GMT to local correction */
quint32 sigfigs; /* accuracy of timestamps */
quint32 snapLen; /* max length of captured packets, in octets */
quint32 network; /* data link type */
} PcapFileHeader;
const quint32 kPcapFileMagic = 0xa1b2c3d4;
const quint32 kPcapFileMagicSwapped = 0xd4c3b2a1;
const quint16 kPcapFileVersionMajor = 2;
const quint16 kPcapFileVersionMinor = 4;
const quint32 kMaxSnapLen = 65535;
typedef struct {
quint32 tsSec; /* timestamp seconds */
quint32 tsUsec; /* timestamp microseconds */
quint32 inclLen; /* number of octets of packet saved in file */
quint32 origLen; /* actual length of packet */
} PcapPacketHeader;
const quint32 kDltEthernet = 1;
PcapFileFormat pcapFileFormat;
@ -75,54 +63,140 @@ PcapFileFormat::~PcapFileFormat()
bool PcapFileFormat::openStreams(const QString fileName,
OstProto::StreamConfigList &streams, QString &error)
{
bool viaPdml = true; // TODO: shd be a param to function
bool isOk = false;
int pktCount;
QFile file(fileName);
QDataStream fd;
QTemporaryFile file2;
quint32 magic;
uchar gzipMagic[2];
int len;
PcapFileHeader fileHdr;
PcapPacketHeader pktHdr;
quint32 len;
int pktCount = 1;
QByteArray pktBuf;
if (!file.open(QIODevice::ReadOnly))
goto _err_open;
if (file.size() < sizeof(fileHdr))
goto _err_truncated;
len = file.peek((char*)gzipMagic, sizeof(gzipMagic));
if (len < int(sizeof(gzipMagic)))
goto _err_reading_magic;
fd.setDevice(&file);
if ((gzipMagic[0] == 0x1f) && (gzipMagic[1] == 0x8b))
{
QProcess gzip;
fd >> magic;
if (!file2.open())
{
error.append("Unable to open temporary file to uncompress .gz\n");
goto _err_unzip_fail;
}
qDebug("decompressing to %s", file2.fileName().toAscii().constData());
gzip.setStandardOutputFile(file2.fileName());
// FIXME: hardcoded prog name
gzip.start("C:/Program Files/CmdLineTools/gzip.exe",
QStringList()
<< "-d"
<< "-c"
<< fileName);
if (!gzip.waitForStarted(-1))
{
error.append(QString("Unable to start gzip\n"));
goto _err_unzip_fail;
}
if (!gzip.waitForFinished(-1))
{
error.append(QString("Error running gzip\n"));
goto _err_unzip_fail;
}
file2.seek(0);
fd_.setDevice(&file2);
}
else
{
fd_.setDevice(&file);
}
fd_ >> magic;
qDebug("magic = %08x", magic);
if (magic == kPcapFileMagicSwapped)
{
// Toggle Byte order
if (fd.byteOrder() == QDataStream::BigEndian)
fd.setByteOrder(QDataStream::LittleEndian);
if (fd_.byteOrder() == QDataStream::BigEndian)
fd_.setByteOrder(QDataStream::LittleEndian);
else
fd.setByteOrder(QDataStream::BigEndian);
fd_.setByteOrder(QDataStream::BigEndian);
}
else if (magic != kPcapFileMagic)
goto _err_bad_magic;
fd >> fileHdr.versionMajor;
fd >> fileHdr.versionMinor;
fd >> fileHdr.thisZone;
fd >> fileHdr.sigfigs;
fd >> fileHdr.snapLen;
fd >> fileHdr.network;
fd_ >> fileHdr.versionMajor;
fd_ >> fileHdr.versionMinor;
fd_ >> fileHdr.thisZone;
fd_ >> fileHdr.sigfigs;
fd_ >> fileHdr.snapLen;
fd_ >> fileHdr.network;
if ((fileHdr.versionMajor != kPcapFileVersionMajor) ||
(fileHdr.versionMinor != kPcapFileVersionMinor))
goto _err_unsupported_version;
// TODO: what do we do about non-ethernet networks?
#if 1
// XXX: we support only Ethernet, for now
if (fileHdr.network != kDltEthernet)
goto _err_unsupported_encap;
#endif
pktBuf.resize(fileHdr.snapLen);
while (!fd.atEnd())
if (viaPdml)
{
QTemporaryFile pdmlFile;
PdmlReader reader(&streams);
QProcess tshark;
if (!pdmlFile.open())
{
error.append("Unable to open temporary file to create PDML\n");
goto _non_pdml;
}
qDebug("generating PDML %s", pdmlFile.fileName().toAscii().constData());
tshark.setStandardOutputFile(pdmlFile.fileName());
// FIXME: hardcoded prog name
tshark.start("C:/Program Files/Wireshark/Tshark.exe",
QStringList()
<< QString("-r%1").arg(fileName)
<< "-Tpdml");
if (!tshark.waitForStarted(-1))
{
error.append(QString("Unable to start tshark\n"));
goto _non_pdml;
}
if (!tshark.waitForFinished(-1))
{
error.append(QString("Error running tshark\n"));
goto _non_pdml;
}
isOk = reader.read(&pdmlFile, this); // TODO: pass error string?
goto _exit;
}
_non_pdml:
pktCount = 1;
while (!fd_.atEnd())
{
OstProto::Stream *stream = streams.add_stream();
OstProto::Protocol *proto = stream->add_protocol();
@ -133,21 +207,11 @@ bool PcapFileFormat::openStreams(const QString fileName,
proto->mutable_protocol_id()->set_id(
OstProto::Protocol::kHexDumpFieldNumber);
// read PcapPacketHeader
fd >> pktHdr.tsSec;
fd >> pktHdr.tsUsec;
fd >> pktHdr.inclLen;
fd >> pktHdr.origLen;
// TODO: chk fd.status()
readPacket(pktHdr, pktBuf);
// validations on inclLen <= origLen && inclLen <= snapLen
Q_ASSERT(pktHdr.inclLen <= fileHdr.snapLen); // TODO: convert to if
// read Pkt contents
len = fd.readRawData(pktBuf.data(), pktHdr.inclLen); // TODO: use while?
Q_ASSERT(len == pktHdr.inclLen); // TODO: remove assert
hexDump->set_content(pktBuf.data(), pktHdr.inclLen);
hexDump->set_pad_until_end(false);
@ -159,6 +223,14 @@ bool PcapFileFormat::openStreams(const QString fileName,
isOk = true;
goto _exit;
#if 1
_err_unsupported_encap:
error = QString(tr("%1 has non-ethernet encapsulation (%2) which is "
"not supported - Sorry!"))
.arg(fileName).arg(fileHdr.network);
goto _exit;
#endif
_err_unsupported_version:
error = QString(tr("%1 is in PCAP version %2.%3 format which is "
"not supported - Sorry!"))
@ -169,24 +241,65 @@ _err_bad_magic:
error = QString(tr("%1 is not a valid PCAP file")).arg(fileName);
goto _exit;
#if 0
_err_truncated:
error = QString(tr("%1 is too short")).arg(fileName);
goto _exit;
#endif
_err_unzip_fail:
goto _exit;
_err_reading_magic:
error = QString(tr("Unable to read magic from %1")).arg(fileName);
goto _exit;
_err_open:
error = QString(tr("Unable to open file: %1")).arg(fileName);
goto _exit;
_exit:
file.close();
return isOk;
}
/*!
Reads packet meta data into pktHdr and packet content into buf.
Returns true if packet is read successfully, false otherwise.
*/
bool PcapFileFormat::readPacket(PcapPacketHeader &pktHdr, QByteArray &pktBuf)
{
quint32 len;
// TODO: chk fd_.status()
// read PcapPacketHeader
fd_ >> pktHdr.tsSec;
fd_ >> pktHdr.tsUsec;
fd_ >> pktHdr.inclLen;
fd_ >> pktHdr.origLen;
// TODO: chk fd_.status()
// XXX: should never be required, but we play safe
if (quint32(pktBuf.size()) < pktHdr.inclLen)
pktBuf.resize(pktHdr.inclLen);
// read Pkt contents
len = fd_.readRawData(pktBuf.data(), pktHdr.inclLen); // TODO: use while?
Q_ASSERT(len == pktHdr.inclLen); // TODO: remove assert
pktBuf.resize(len);
return true;
}
bool PcapFileFormat::saveStreams(const OstProto::StreamConfigList streams,
const QString fileName, QString &error)
{
bool isOk = false;
QFile file(fileName);
QDataStream fd;
PcapFileHeader fileHdr;
PcapPacketHeader pktHdr;
QByteArray pktBuf;
@ -194,7 +307,7 @@ bool PcapFileFormat::saveStreams(const OstProto::StreamConfigList streams,
if (!file.open(QIODevice::WriteOnly))
goto _err_open;
fd.setDevice(&file);
fd_.setDevice(&file);
fileHdr.magicNumber = kPcapFileMagic;
fileHdr.versionMajor = kPcapFileVersionMajor;
@ -202,15 +315,15 @@ bool PcapFileFormat::saveStreams(const OstProto::StreamConfigList streams,
fileHdr.thisZone = 0;
fileHdr.sigfigs = 0;
fileHdr.snapLen = kMaxSnapLen;
fileHdr.network = 1; // Ethernet; FIXME: Hardcoding
fileHdr.network = kDltEthernet;
fd << fileHdr.magicNumber;
fd << fileHdr.versionMajor;
fd << fileHdr.versionMinor;
fd << fileHdr.thisZone;
fd << fileHdr.sigfigs;
fd << fileHdr.snapLen;
fd << fileHdr.network;
fd_ << fileHdr.magicNumber;
fd_ << fileHdr.versionMajor;
fd_ << fileHdr.versionMinor;
fd_ << fileHdr.thisZone;
fd_ << fileHdr.sigfigs;
fd_ << fileHdr.snapLen;
fd_ << fileHdr.network;
pktBuf.resize(kMaxSnapLen);
@ -230,11 +343,11 @@ bool PcapFileFormat::saveStreams(const OstProto::StreamConfigList streams,
if (pktHdr.inclLen > fileHdr.snapLen)
pktHdr.inclLen = fileHdr.snapLen;
fd << pktHdr.tsSec;
fd << pktHdr.tsUsec;
fd << pktHdr.inclLen;
fd << pktHdr.origLen;
fd.writeRawData(pktBuf.data(), pktHdr.inclLen);
fd_ << pktHdr.tsSec;
fd_ << pktHdr.tsUsec;
fd_ << pktHdr.inclLen;
fd_ << pktHdr.origLen;
fd_.writeRawData(pktBuf.data(), pktHdr.inclLen);
}
file.close();

View File

@ -1,5 +1,5 @@
/*
Copyright (C) 2010 Srivats P.
Copyright (C) 2011 Srivats P.
This file is part of "Ostinato"
@ -21,12 +21,16 @@ along with this program. If not, see <http://www.gnu.org/licenses/>
#include "protocol.pb.h"
#include <QDataStream>
#include <QObject>
class PdmlReader;
class PcapFileFormat : public QObject
{
Q_OBJECT
friend class PdmlReader;
public:
PcapFileFormat();
~PcapFileFormat();
@ -35,6 +39,28 @@ public:
OstProto::StreamConfigList &streams, QString &error);
bool saveStreams(const OstProto::StreamConfigList streams,
const QString fileName, QString &error);
private:
typedef struct {
quint32 magicNumber; /* magic number */
quint16 versionMajor; /* major version number */
quint16 versionMinor; /* minor version number */
qint32 thisZone; /* GMT to local correction */
quint32 sigfigs; /* accuracy of timestamps */
quint32 snapLen; /* max length of captured packets, in octets */
quint32 network; /* data link type */
} PcapFileHeader;
typedef struct {
quint32 tsSec; /* timestamp seconds */
quint32 tsUsec; /* timestamp microseconds */
quint32 inclLen; /* number of octets of packet saved in file */
quint32 origLen; /* actual length of packet */
} PcapPacketHeader;
bool readPacket(PcapPacketHeader &pktHdr, QByteArray &pktBuf);
QDataStream fd_;
};
extern PcapFileFormat pcapFileFormat;

View File

@ -19,8 +19,10 @@ along with this program. If not, see <http://www.gnu.org/licenses/>
#include "pdml_p.h"
#include "PcapFileFormat.h"
#include "mac.pb.h"
#include "eth2.pb.h"
#include "dot3.pb.h"
#include "hexdump.pb.h"
#include "ip4.pb.h"
#include "ip6.pb.h"
@ -349,6 +351,7 @@ bool PdmlParser::fatalError(const QXmlParseException &exception)
PdmlReader::PdmlReader(OstProto::StreamConfigList *streams)
{
gPdmlReader = this;
pcap_ = NULL;
pass_ = 0;
streams_ = streams;
@ -358,20 +361,21 @@ PdmlReader::PdmlReader(OstProto::StreamConfigList *streams)
#if 0
factory_.insert("fake-field-wrapper",
new PdmlFakeFieldWrapperProtocol());
#endif
factory_.insert("eth",PdmlEthProtocol::createInstance);
factory_.insert("ip",PdmlIp4Protocol::createInstance);
factory_.insert("ipv6",PdmlIp6Protocol::createInstance);
factory_.insert("tcp",PdmlTcpProtocol::createInstance);
#endif
}
PdmlReader::~PdmlReader()
{
}
bool PdmlReader::read(QIODevice *device)
bool PdmlReader::read(QIODevice *device, PcapFileFormat *pcap)
{
setDevice(device);
pcap_ = pcap;
packetCount_ = 0;
// 1st pass - preprocessing fake fields
@ -408,7 +412,14 @@ bool PdmlReader::read(QIODevice *device)
}
}
return !error();
if (error())
{
qDebug("Line %lld", lineNumber());
qDebug("Col %lld", columnNumber());
qDebug("%s", errorString().toAscii().constData()); // FIXME
return false;
}
return true;
}
// TODO: use a temp pool to avoid a lot of new/delete
@ -502,6 +513,8 @@ void PdmlReader::readPdml()
}
}
/////////////////////// PASS 1 //////////////////////////
void PdmlReader::readPacketPass1()
{
Q_ASSERT(isStartElement() && name() == "packet");
@ -614,17 +627,24 @@ void PdmlReader::readFieldPass1()
}
}
/////////////////////// PASS 2 //////////////////////////
void PdmlReader::readPacket()
{
//Q_ASSERT(isStartElement() && name() == "packet");
PcapFileFormat::PcapPacketHeader pktHdr;
Q_ASSERT(isStartElement() && name() == "packet");
qDebug("%s: packetNum = %d", __FUNCTION__, packetCount_);
// XXX: For now, each packet is converted to a stream
// XXX: we play dumb and convert each packet to a stream, for now
currentStream_ = streams_->add_stream();
currentStream_->mutable_stream_id()->set_id(packetCount_);
currentStream_->mutable_core()->set_is_enabled(true);
if (pcap_)
pcap_->readPacket(pktHdr, pktBuf_);
while (!atEnd())
{
readNext();
@ -668,9 +688,23 @@ void PdmlReader::readProto()
Q_ASSERT(isStartElement() && name() == "proto");
QString protoName = attributes().value("name").toString();
qDebug("proto: %s", protoName.toAscii().constData());
int pos = -1;
#if 0
if (!attributes().value("pos").isEmpty())
pos = attributes().value("pos").toString().toInt();
qDebug("proto: %s, pos = %d", protoName.toAscii().constData(), pos);
// This is a heuristic to skip protocols which are not part of
// this frame, but of a reassembled segment spanning several frames
if ((pos == 0) && (currentStream_->protocol_size() > 0))
{
qDebug("(skipped)");
skipElement();
return;
}
#if 1
if (protoName.isEmpty() || (protoName == "expert"))
{
skipElement();
@ -678,6 +712,34 @@ void PdmlReader::readProto()
}
#endif
if (!factory_.contains(protoName) && pcap_)
{
int pos = -1;
int size = -1;
if (!attributes().value("pos").isEmpty())
pos = attributes().value("pos").toString().toInt();
if (!attributes().value("size").isEmpty())
size = attributes().value("size").toString().toInt();
if (pos >= 0 && size > 0
&& ((pos + size) <= pktBuf_.size()))
{
OstProto::Protocol *proto = currentStream_->add_protocol();
OstProto::HexDump *hexDump = proto->MutableExtension(
OstProto::hexDump);
proto->mutable_protocol_id()->set_id(
OstProto::Protocol::kHexDumpFieldNumber);
hexDump->set_content(pktBuf_.constData() + pos, size);
hexDump->set_pad_until_end(false);
skipElement();
return;
}
}
pdmlProto = allocPdmlProtocol(protoName);
Q_ASSERT(pdmlProto != NULL);
@ -902,6 +964,11 @@ void PdmlUnknownProtocol::preProtocolHandler(QString name,
if (!isOk)
goto _skip_pos_size_proc;
// If pos+size goes beyond the frame length, this is a "reassembled"
// protocol and should be skipped
if (quint32(pos + size) > stream->core().frame_len())
goto _skip_pos_size_proc;
expPos_ = pos;
endPos_ = expPos_ + size;
@ -1048,6 +1115,8 @@ void PdmlFakeFieldWrapperProtocol::postProtocolHandler(OstProto::Stream *stream)
qDebug("%s: expPos_ = %d\n", __FUNCTION__, expPos_);
// TODO: if content size is zero, remove protocol?
hexDump->set_pad_until_end(false);
expPos_ = -1;
}
@ -1059,6 +1128,7 @@ void PdmlFakeFieldWrapperProtocol::unknownFieldHandler(QString name, int pos,
stream->protocol_size()-1)->MutableExtension(OstProto::hexDump);
if ((pos == expPos_) && (size >= 0) &&
(!name.startsWith("tcp.segment")) &&
(!attributes.value("unmaskedvalue").isEmpty() ||
!attributes.value("value").isEmpty()))
{
@ -1106,6 +1176,36 @@ void PdmlEthProtocol::unknownFieldHandler(QString name, int pos, int size,
eth2->set_type(attributes.value("value").toString().toUInt(&isOk, kBaseHex));
eth2->set_is_override_type(true);
}
else if (name == "eth.len")
{
OstProto::Protocol *proto = stream->add_protocol();
proto->mutable_protocol_id()->set_id(
OstProto::Protocol::kDot3FieldNumber);
OstProto::Dot3 *dot3 = proto->MutableExtension(OstProto::dot3);
bool isOk;
dot3->set_length(attributes.value("value").toString().toUInt(&isOk, kBaseHex));
dot3->set_is_override_length(true);
}
else if (name == "eth.trailer")
{
QByteArray trailer = QByteArray::fromHex(
attributes.value("value").toString().toUtf8());
stream->mutable_core()->mutable_name()->append(trailer.constData(),
trailer.size());
}
else if ((name == "eth.fcs") ||
attributes.value("show").toString().startsWith("Frame check sequence"))
{
QByteArray trailer = QByteArray::fromHex(
attributes.value("value").toString().toUtf8());
stream->mutable_core()->mutable_name()->append(trailer.constData(),
trailer.size());
}
}
@ -1141,7 +1241,13 @@ void PdmlIp4Protocol::unknownFieldHandler(QString name, int pos, int size,
{
bool isOk;
if (name == "ip.flags")
if ((name == "ip.options") ||
attributes.value("show").toString().startsWith("Options"))
{
options_ = QByteArray::fromHex(
attributes.value("value").toString().toUtf8());
}
else if (name == "ip.flags")
{
OstProto::Ip4 *ip4 = stream->mutable_protocol(
stream->protocol_size()-1)->MutableExtension(OstProto::ip4);
@ -1160,6 +1266,21 @@ void PdmlIp4Protocol::postProtocolHandler(OstProto::Stream *stream)
ip4->set_is_override_totlen(true); // FIXME
ip4->set_is_override_proto(true); // FIXME
ip4->set_is_override_cksum(true); // FIXME
if (options_.size())
{
OstProto::Protocol *proto = stream->add_protocol();
proto->mutable_protocol_id()->set_id(
OstProto::Protocol::kHexDumpFieldNumber);
OstProto::HexDump *hexDump = proto->MutableExtension(OstProto::hexDump);
hexDump->mutable_content()->append(options_.constData(),
options_.size());
hexDump->set_pad_until_end(false);
options_.resize(0);
}
}
// ---------------------------------------------------------- //
@ -1255,8 +1376,8 @@ void PdmlTcpProtocol::unknownFieldHandler(QString name, int pos, int size,
&& attributes.value("show").toString().startsWith("TCP segment data"))
{
segmentData_ = QByteArray::fromHex(attributes.value("value").toString().toUtf8());
stream->mutable_core()->mutable_name()->append(segmentData_.constData(),
segmentData_.size());
stream->mutable_core()->mutable_name()->insert(0,
segmentData_.constData(), segmentData_.size());
}
}

View File

@ -91,6 +91,7 @@ private:
#endif
class PdmlUnknownProtocol;
class PcapFileFormat;
class PdmlReader : public QXmlStreamReader
{
friend class PdmlUnknownProtocol;
@ -98,7 +99,7 @@ public:
PdmlReader(OstProto::StreamConfigList *streams);
~PdmlReader();
bool read(QIODevice *device);
bool read(QIODevice *device, PcapFileFormat *pcap = NULL);
private:
PdmlDefaultProtocol* allocPdmlProtocol(QString protoName);
@ -144,12 +145,15 @@ private:
QMap<QString, FactoryMethod> factory_;
OstProto::StreamConfigList *streams_;
PcapFileFormat *pcap_;
int pass_;
int packetCount_;
OstProto::Stream *currentStream_;
QList<Fragment> pktFragments_;
QByteArray pktBuf_;
//PdmlDefaultProtocol *currentPdmlProtocol_;
//google::protobuf::Message *currentProtocolMsg_;
};
@ -230,6 +234,8 @@ public:
virtual void unknownFieldHandler(QString name, int pos, int size,
const QXmlStreamAttributes &attributes, OstProto::Stream *stream);
virtual void postProtocolHandler(OstProto::Stream *stream);
private:
QByteArray options_;
};
class PdmlIp6Protocol : public PdmlDefaultProtocol

View File

@ -26,15 +26,17 @@ int main(int argc, char* argv[])
QString inFile(argv[1]);
QString outFile(argv[2]);
if (!pdmlFileFormat.openStreams(inFile, streams, error))
if (!pcapFileFormat.openStreams(inFile, streams, error))
{
fprintf(stdout, "failed reading streams from %s\n", inFile.toAscii().constData());
fprintf(stdout, "failed reading streams from %s:%s\n",
inFile.toAscii().constData(), error.toAscii().constData());
exit(1);
}
if (!pcapFileFormat.saveStreams(streams, outFile, error))
{
fprintf(stdout, "failed writing streams to %s\n", outFile.toAscii().constData());
fprintf(stdout, "failed writing streams to %s:%s\n",
outFile.toAscii().constData(), error.toAscii().constData());
exit(1);
}