diff --git a/common/ostfile.pro b/common/ostfile.pro index de7eef7..01e31ff 100644 --- a/common/ostfile.pro +++ b/common/ostfile.pro @@ -24,6 +24,9 @@ HEADERS = \ sessionfileformat.h \ streamfileformat.h +HEADERS += \ + pcapfileformatoptions.h + SOURCES += \ ostprotolib.cpp \ nativefileformat.cpp \ @@ -39,6 +42,9 @@ SOURCES += \ streamfileformat.cpp \ spinboxdelegate.cpp +SOURCES += \ + pcapfileformatoptions.cpp + SOURCES += \ vlanpdml.cpp \ svlanpdml.cpp \ diff --git a/common/pcapfileformat.cpp b/common/pcapfileformat.cpp index f3ceeba..a865f4d 100644 --- a/common/pcapfileformat.cpp +++ b/common/pcapfileformat.cpp @@ -19,6 +19,8 @@ along with this program. If not, see #include "pcapfileformat.h" +#include "pcapfileformatoptions.h" + #include "pdmlreader.h" #include "ostprotolib.h" #include "streambase.h" @@ -42,38 +44,6 @@ const quint32 kDltEthernet = 1; PcapFileFormat pcapFileFormat; -PcapImportOptionsDialog::PcapImportOptionsDialog(QVariantMap *options) - : QDialog(NULL) -{ - setupUi(this); - setAttribute(Qt::WA_DeleteOnClose); - options_ = options; - - viaPdml->setChecked(options_->value("ViaPdml").toBool()); - // XXX: By default this key is absent - so that pcap import tests - // evaluate to false and hence show minimal diffs. - // However, for the GUI user, this should be enabled by default. - recalculateCksums->setChecked( - options_->value("RecalculateCksums", QVariant(true)) - .toBool()); - doDiff->setChecked(options_->value("DoDiff").toBool()); - - connect(buttonBox, SIGNAL(accepted()), this, SLOT(accept())); -} - -PcapImportOptionsDialog::~PcapImportOptionsDialog() -{ -} - -void PcapImportOptionsDialog::accept() -{ - options_->insert("ViaPdml", viaPdml->isChecked()); - options_->insert("RecalculateCksums", recalculateCksums->isChecked()); - options_->insert("DoDiff", doDiff->isChecked()); - - QDialog::accept(); -} - PcapFileFormat::PcapFileFormat() { importOptions_.insert("ViaPdml", true); diff --git a/common/pcapfileformat.h b/common/pcapfileformat.h index 5e8a744..5b5bf6a 100644 --- a/common/pcapfileformat.h +++ b/common/pcapfileformat.h @@ -20,24 +20,10 @@ along with this program. If not, see #define _PCAP_FILE_FORMAT_H #include "streamfileformat.h" -#include "ui_pcapfileimport.h" #include #include -class PcapImportOptionsDialog: public QDialog, public Ui::PcapFileImport -{ -public: - PcapImportOptionsDialog(QVariantMap *options); - ~PcapImportOptionsDialog(); - -private slots: - void accept(); - -private: - QVariantMap *options_; -}; - class PdmlReader; class PcapFileFormat : public StreamFileFormat { diff --git a/common/pcapfileformatoptions.cpp b/common/pcapfileformatoptions.cpp index f3ceeba..f827ac7 100644 --- a/common/pcapfileformatoptions.cpp +++ b/common/pcapfileformatoptions.cpp @@ -17,30 +17,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see */ -#include "pcapfileformat.h" - -#include "pdmlreader.h" -#include "ostprotolib.h" -#include "streambase.h" -#include "hexdump.pb.h" - -#include -#include -#include -#include -#include -#include - -const quint32 kPcapFileMagic = 0xa1b2c3d4; -const quint32 kPcapFileMagicSwapped = 0xd4c3b2a1; -const quint32 kNanoSecondPcapFileMagic = 0xa1b23c4d; -const quint32 kNanoSecondPcapFileMagicSwapped = 0x4d3cb2a1; -const quint16 kPcapFileVersionMajor = 2; -const quint16 kPcapFileVersionMinor = 4; -const quint32 kMaxSnapLen = 65535; -const quint32 kDltEthernet = 1; - -PcapFileFormat pcapFileFormat; +#include "pcapfileformatoptions.h" PcapImportOptionsDialog::PcapImportOptionsDialog(QVariantMap *options) : QDialog(NULL) @@ -74,678 +51,3 @@ void PcapImportOptionsDialog::accept() QDialog::accept(); } -PcapFileFormat::PcapFileFormat() -{ - importOptions_.insert("ViaPdml", true); - importOptions_.insert("DoDiff", true); -} - -PcapFileFormat::~PcapFileFormat() -{ -} - -bool PcapFileFormat::open(const QString fileName, - OstProto::StreamConfigList &streams, QString &error) -{ - bool isOk = false; - QFile file(fileName); - QTemporaryFile file2; - quint32 magic; - uchar gzipMagic[2]; - bool nsecResolution = false; - int len; - PcapFileHeader fileHdr; - PcapPacketHeader pktHdr; - OstProto::Stream *prevStream = NULL; - quint64 lastXsec = 0; - int pktCount; - qint64 byteCount = 0; - qint64 byteTotal; - QByteArray pktBuf; - bool tryConvert = true; - - if (!file.open(QIODevice::ReadOnly)) - goto _err_open; - - len = file.peek((char*)gzipMagic, sizeof(gzipMagic)); - if (len < int(sizeof(gzipMagic))) - goto _err_reading_magic; - - if ((gzipMagic[0] == 0x1f) && (gzipMagic[1] == 0x8b)) - { - QProcess gzip; - - emit status("Decompressing..."); - emit target(0); - - if (!file2.open()) - { - error.append("Unable to open temporary file to uncompress .gz\n"); - goto _err_unzip_fail; - } - - qDebug("decompressing to %s", qPrintable(file2.fileName())); - - gzip.setStandardOutputFile(file2.fileName()); - gzip.start(OstProtoLib::gzipPath(), - QStringList() - << "-d" - << "-c" - << fileName); - if (!gzip.waitForStarted(-1)) - { - error.append(QString("Unable to start gzip. Check path in Preferences.\n")); - goto _err_unzip_fail; - } - - if (!gzip.waitForFinished(-1)) - { - error.append(QString("Error running gzip\n")); - goto _err_unzip_fail; - } - - file2.seek(0); - - fd_.setDevice(&file2); - } - else - { - fd_.setDevice(&file); - } - -_retry: - byteTotal = fd_.device()->size() - sizeof(fileHdr); - - emit status("Reading File Header..."); - emit target(0); - - fd_ >> magic; - - qDebug("magic = %08x", magic); - - if (magic == kPcapFileMagic) - { - // Do nothing - } - else if (magic == kNanoSecondPcapFileMagic) - { - nsecResolution = true; - } - else if ((magic == kPcapFileMagicSwapped) - || (magic == kNanoSecondPcapFileMagicSwapped)) - { - // Toggle Byte order - if (fd_.byteOrder() == QDataStream::BigEndian) - fd_.setByteOrder(QDataStream::LittleEndian); - else - fd_.setByteOrder(QDataStream::BigEndian); - - nsecResolution = (magic == kNanoSecondPcapFileMagicSwapped); - } - else // Not a pcap file (could be pcapng or something else) - { - if (tryConvert) - { - // Close and reopen the temp file to be safe - file2.close(); - if (!file2.open()) - { - error.append("Unable to open temporary file to convert to PCAP\n"); - goto _err_convert2pcap; - } - fd_.setDevice(0); // disconnect data stream from file - - if (convertToStandardPcap(fileName, file2.fileName(), error)) - { - fd_.setDevice(&file2); - tryConvert = false; - goto _retry; - } - else - { - error = QString(tr("Unable to convert %1 to standard PCAP format")) - .arg(fileName); - goto _err_convert2pcap; - } - } - else - goto _err_bad_magic; - } - - qDebug("reading filehdr"); - - fd_ >> fileHdr.versionMajor; - fd_ >> fileHdr.versionMinor; - fd_ >> fileHdr.thisZone; - fd_ >> fileHdr.sigfigs; - fd_ >> fileHdr.snapLen; - fd_ >> fileHdr.network; - - qDebug("version check"); - if ((fileHdr.versionMajor != kPcapFileVersionMajor) || - (fileHdr.versionMinor != kPcapFileVersionMinor)) - goto _err_unsupported_version; - -#if 1 - // XXX: we support only Ethernet, for now - if (fileHdr.network != kDltEthernet) - goto _err_unsupported_encap; -#endif - - pktBuf.resize(fileHdr.snapLen); - - // XXX: PDML also needs the PCAP file to cross check packet bytes - // with the PDML data, so we can't do PDML conversion any earlier - // than this - qDebug("pdml check"); - if (importOptions_.value("ViaPdml").toBool()) - { - QProcess tshark; - QTemporaryFile pdmlFile; - PdmlReader reader(&streams, importOptions_); - - if (!pdmlFile.open()) - { - error.append("Unable to open temporary file to create PDML\n"); - goto _non_pdml; - } - - qDebug("generating PDML %s", qPrintable(pdmlFile.fileName())); - emit status("Generating PDML..."); - emit target(0); - - tshark.setStandardOutputFile(pdmlFile.fileName()); - tshark.start(OstProtoLib::tsharkPath(), - QStringList() - << QString("-r%1").arg(fileName) - << "-otcp.desegment_tcp_streams:FALSE" - << "-Tpdml"); - if (!tshark.waitForStarted(-1)) - { - error.append(QString("Unable to start tshark. Check path in preferences.\n")); - goto _non_pdml; - } - - if (!tshark.waitForFinished(-1)) - { - error.append(QString("Error running tshark\n")); - goto _non_pdml; - } - - connect(&reader, SIGNAL(progress(int)), this, SIGNAL(progress(int))); - - emit status("Reading PDML packets..."); - emit target(100); // in percentage - - // pdml reader needs pcap, so pass self - isOk = reader.read(&pdmlFile, this, &stop_); - - if (stop_) - goto _user_cancel; - - if (!isOk) - { - error.append(QString("Error processing PDML (%1, %2): %3\n") - .arg(reader.lineNumber()) - .arg(reader.columnNumber()) - .arg(reader.errorString())); - goto _exit; - } - - if (!importOptions_.value("DoDiff").toBool()) - goto _exit; - - - // !-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-! - // Let's do the diff ... - // !-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-! - - QProcess awk; - QProcess diff; - QTemporaryFile originalTextFile; - QTemporaryFile importedPcapFile; - QTemporaryFile importedTextFile; - QTemporaryFile diffFile; - const QString kAwkFilter = - "/^[^0]/ { " - "printf \" %s \", $1;" - "for (i=4; i %s", - qPrintable(originalTextFile.fileName()), - qPrintable(importedTextFile.fileName()), - qPrintable(diffFile.fileName())); - - emit status("Taking diff..."); - emit target(0); - - diff.setStandardOutputFile(diffFile.fileName()); - diff.start(OstProtoLib::diffPath(), - QStringList() - << "-u" - << "-F^ [1-9]" - << QString("--label=%1 (actual)") - .arg(QFileInfo(fileName).fileName()) - << QString("--label=%1 (imported)") - .arg(QFileInfo(fileName).fileName()) - << originalTextFile.fileName() - << importedTextFile.fileName()); - if (!diff.waitForStarted(-1)) - { - error.append(QString("Unable to start diff. Check path in Preferences.\n") - .arg(diff.exitCode())); - goto _diff_fail; - } - - if (!diff.waitForFinished(-1)) - { - error.append(QString("Error running diff\n")); - goto _diff_fail; - } - - diffFile.close(); - if (diffFile.size()) - { - error.append(tr("

There is a diff between the original and imported streams. See details to review the diff.

Why a diff? See possible reasons.

\n\n\n\n").arg("https://jump.ostinato.org/pcapdiff")); - diffFile.open(); - diffFile.seek(0); - error.append(QString(diffFile.readAll())); - } - - goto _exit; - } - -_non_pdml: - qDebug("pcap resolution: %s", nsecResolution ? "nsec" : "usec"); - emit status("Reading Packets..."); - emit target(100); // in percentage - pktCount = 1; - while (!fd_.atEnd()) - { - OstProto::Stream *stream = streams.add_stream(); - OstProto::Protocol *proto = stream->add_protocol(); - OstProto::HexDump *hexDump = proto->MutableExtension(OstProto::hexDump); - - proto->mutable_protocol_id()->set_id( - OstProto::Protocol::kHexDumpFieldNumber); - - readPacket(pktHdr, pktBuf); - - // validations on inclLen <= origLen && inclLen <= snapLen - Q_ASSERT(pktHdr.inclLen <= fileHdr.snapLen); // TODO: convert to if - - hexDump->set_content(pktBuf.data(), pktHdr.inclLen); - hexDump->set_pad_until_end(false); - - stream->mutable_stream_id()->set_id(pktCount); - stream->mutable_core()->set_is_enabled(true); - stream->mutable_core()->set_frame_len(pktHdr.inclLen+4); // FCS - - stream->mutable_control()->set_num_packets(1); - - // setup packet rate to the timing in pcap (as close as possible) - // use quint64 rather than double to store micro/nano second as - // it has a larger range (~580 years) and therefore better accuracy - const quint64 kXsecsInSec = nsecResolution ? 1e9 : 1e6; - quint64 xsec = (pktHdr.tsSec*kXsecsInSec + pktHdr.tsUsec); - quint64 delta = xsec - lastXsec; - qDebug("pktCount = %d, delta = %llu", pktCount, delta); - - if ((pktCount != 1) && delta) - stream->mutable_control()->set_packets_per_sec(double(kXsecsInSec)/delta); - - if (prevStream) - prevStream->mutable_control()->CopyFrom(stream->control()); - - lastXsec = xsec; - prevStream = stream; - pktCount++; - byteCount += pktHdr.inclLen + sizeof(pktHdr); - emit progress(int(byteCount*100/byteTotal)); // in percentage - if (stop_) - goto _user_cancel; - } - - isOk = true; - goto _exit; - -_user_cancel: - isOk = true; - goto _exit; - -_diff_fail: - goto _exit; - -_err_unsupported_encap: - error = QString(tr("%1 has non-ethernet encapsulation (%2) which is " - "not supported - Sorry!")) - .arg(QFileInfo(fileName).fileName()).arg(fileHdr.network); - goto _exit; - -_err_unsupported_version: - error = QString(tr("%1 is in PCAP version %2.%3 format which is " - "not supported - Sorry!")) - .arg(fileName).arg(fileHdr.versionMajor).arg(fileHdr.versionMinor); - goto _exit; - -_err_bad_magic: - error = QString(tr("%1 is not a valid PCAP file")).arg(fileName); - goto _exit; - -#if 0 -_err_truncated: - error = QString(tr("%1 is too short")).arg(fileName); - goto _exit; -#endif - -_err_unzip_fail: - goto _exit; - -_err_reading_magic: - error = QString(tr("Unable to read magic from %1")).arg(fileName); - goto _exit; - -_err_convert2pcap: - goto _exit; - -_err_open: - error = QString(tr("Unable to open file: %1")).arg(fileName); - goto _exit; - -_exit: - if (!error.isEmpty()) - qDebug("%s", qPrintable(error)); - file.close(); - return isOk; -} - -/*! - Converts a non-PCAP capture file to standard PCAP file format using tshark - - Returns true if conversion was successful, false otherwise. -*/ -bool PcapFileFormat::convertToStandardPcap( - QString fileName, QString outputFileName, QString &error) -{ - qDebug("converting to PCAP %s", qPrintable(outputFileName)); - emit status("Unsupported format. Converting to standard PCAP format..."); - emit target(0); - - QProcess tshark; - tshark.start(OstProtoLib::tsharkPath(), - QStringList() - << QString("-r%1").arg(fileName) - << "-Fnsecpcap" - << QString("-w%1").arg(outputFileName)); - if (!tshark.waitForStarted(-1)) - { - error.append(QString("Unable to start tshark. Check path in preferences.\n")); - return false; - } - - if (!tshark.waitForFinished(-1)) - { - error.append(QString("Error running tshark\n")); - return false; - } - - return true; -} - -/*! - Reads packet meta data into pktHdr and packet content into buf. - - Returns true if packet is read successfully, false otherwise. -*/ -bool PcapFileFormat::readPacket(PcapPacketHeader &pktHdr, QByteArray &pktBuf) -{ - quint32 len; - - // TODO: chk fd_.status() - - // read PcapPacketHeader - fd_ >> pktHdr.tsSec; - fd_ >> pktHdr.tsUsec; - fd_ >> pktHdr.inclLen; - fd_ >> pktHdr.origLen; - - // TODO: chk fd_.status() - - // XXX: should never be required, but we play safe - if (quint32(pktBuf.size()) < pktHdr.inclLen) - pktBuf.resize(pktHdr.inclLen); - - // read Pkt contents - len = fd_.readRawData(pktBuf.data(), pktHdr.inclLen); // TODO: use while? - - Q_ASSERT(len == pktHdr.inclLen); // TODO: remove assert - pktBuf.resize(len); - - return true; -} - -bool PcapFileFormat::save(const OstProto::StreamConfigList streams, - const QString fileName, QString &error) -{ - bool isOk = false; - QFile file(fileName); - PcapFileHeader fileHdr; - PcapPacketHeader pktHdr; - QByteArray pktBuf; - - if (!file.open(QIODevice::WriteOnly)) - goto _err_open; - - fd_.setDevice(&file); - - fileHdr.magicNumber = kNanoSecondPcapFileMagic; - fileHdr.versionMajor = kPcapFileVersionMajor; - fileHdr.versionMinor = kPcapFileVersionMinor; - fileHdr.thisZone = 0; - fileHdr.sigfigs = 0; - fileHdr.snapLen = kMaxSnapLen; - fileHdr.network = kDltEthernet; - - fd_ << fileHdr.magicNumber; - fd_ << fileHdr.versionMajor; - fd_ << fileHdr.versionMinor; - fd_ << fileHdr.thisZone; - fd_ << fileHdr.sigfigs; - fd_ << fileHdr.snapLen; - fd_ << fileHdr.network; - - pktBuf.resize(kMaxSnapLen); - - emit status("Writing Packets..."); - emit target(streams.stream_size()); - - pktHdr.tsSec = 0; - pktHdr.tsUsec = 0; - for (int i = 0; i < streams.stream_size(); i++) - { - StreamBase s; - - s.setId(i); - s.protoDataCopyFrom(streams.stream(i)); - // TODO: expand frameIndex for each stream - s.frameValue((uchar*)pktBuf.data(), pktBuf.size(), 0); - - pktHdr.inclLen = s.frameProtocolLength(0); // FIXME: stream index = 0 - pktHdr.origLen = s.frameLen() - 4; // FCS; FIXME: Hardcoding - - qDebug("savepcap i=%d, incl/orig len = %d/%d", i, - pktHdr.inclLen, pktHdr.origLen); - - if (pktHdr.inclLen > fileHdr.snapLen) - pktHdr.inclLen = fileHdr.snapLen; - - fd_ << pktHdr.tsSec; - fd_ << pktHdr.tsUsec; - fd_ << pktHdr.inclLen; - fd_ << pktHdr.origLen; - fd_.writeRawData(pktBuf.data(), pktHdr.inclLen); - - if (s.packetRate()) - { - quint64 delta = quint64(1e9/s.packetRate()); - pktHdr.tsSec += delta/quint32(1e9); - pktHdr.tsUsec += delta % quint32(1e9); - } - - if (pktHdr.tsUsec >= quint32(1e9)) - { - pktHdr.tsSec++; - pktHdr.tsUsec -= quint32(1e9); - } - - emit progress(i); - } - - file.close(); - - isOk = true; - goto _exit; - -_err_open: - error = QString(tr("Unable to open file: %1")).arg(fileName); - goto _exit; - -_exit: - return isOk; -} - -QDialog* PcapFileFormat::openOptionsDialog() -{ - return new PcapImportOptionsDialog(&importOptions_); -} - -bool PcapFileFormat::isMyFileFormat(const QString /*fileName*/) -{ - // TODO - return true; -} - -bool PcapFileFormat::isMyFileType(const QString fileType) -{ - if (fileType.startsWith("PCAP")) - return true; - else - return false; -} diff --git a/common/pcapfileformatoptions.h b/common/pcapfileformatoptions.h index 5e8a744..6fddce7 100644 --- a/common/pcapfileformatoptions.h +++ b/common/pcapfileformatoptions.h @@ -16,13 +16,11 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see */ -#ifndef _PCAP_FILE_FORMAT_H -#define _PCAP_FILE_FORMAT_H +#ifndef _PCAP_FILE_FORMAT_OPTIONS_H +#define _PCAP_FILE_FORMAT_OPTIONS_H -#include "streamfileformat.h" #include "ui_pcapfileimport.h" -#include #include class PcapImportOptionsDialog: public QDialog, public Ui::PcapFileImport @@ -38,51 +36,4 @@ private: QVariantMap *options_; }; -class PdmlReader; -class PcapFileFormat : public StreamFileFormat -{ - friend class PdmlReader; - -public: - PcapFileFormat(); - ~PcapFileFormat(); - - bool open(const QString fileName, - OstProto::StreamConfigList &streams, QString &error); - bool save(const OstProto::StreamConfigList streams, - const QString fileName, QString &error); - - virtual QDialog* openOptionsDialog(); - - bool isMyFileFormat(const QString fileName); - bool isMyFileType(const QString fileType); - -private: - typedef struct { - quint32 magicNumber; /* magic number */ - quint16 versionMajor; /* major version number */ - quint16 versionMinor; /* minor version number */ - qint32 thisZone; /* GMT to local correction */ - quint32 sigfigs; /* accuracy of timestamps */ - quint32 snapLen; /* max length of captured packets, in octets */ - quint32 network; /* data link type */ - } PcapFileHeader; - - typedef struct { - quint32 tsSec; /* timestamp seconds */ - quint32 tsUsec; /* timestamp microseconds */ - quint32 inclLen; /* number of octets of packet saved in file */ - quint32 origLen; /* actual length of packet */ - } PcapPacketHeader; - - bool convertToStandardPcap(QString fileName, QString outputFileName, - QString &error); - bool readPacket(PcapPacketHeader &pktHdr, QByteArray &pktBuf); - - QDataStream fd_; - QVariantMap importOptions_; -}; - -extern PcapFileFormat pcapFileFormat; - #endif