| #!/usr/bin/env python3 |
| # IPFIX support for Scapy (RFC7011) |
| |
| from scapy.all import ( |
| bind_layers, |
| FieldLenField, |
| IntField, |
| Packet, |
| PacketListField, |
| ShortEnumField, |
| ShortField, |
| StrLenField, |
| ) |
| from scapy.layers.inet import UDP |
| |
| |
| # IPFIX Information Elements http://www.iana.org/assignments/ipfix/ipfix.xhtml |
| information_elements = { |
| 1: "octetDeltaCount", |
| 2: "packetDeltaCount", |
| 3: "deltaFlowCount", |
| 4: "protocolIdentifier", |
| 5: "ipClassOfService", |
| 6: "tcpControlBits", |
| 7: "sourceTransportPort", |
| 8: "sourceIPv4Address", |
| 9: "sourceIPv4PrefixLength", |
| 10: "ingressInterface", |
| 11: "destinationTransportPort", |
| 12: "destinationIPv4Address", |
| 13: "destinationIPv4PrefixLength", |
| 14: "egressInterface", |
| 15: "ipNextHopIPv4Address", |
| 16: "bgpSourceAsNumber", |
| 17: "bgpDestinationAsNumber", |
| 18: "bgpNextHopIPv4Address", |
| 19: "postMCastPacketDeltaCount", |
| 20: "postMCastOctetDeltaCount", |
| 21: "flowEndSysUpTime", |
| 22: "flowStartSysUpTime", |
| 23: "postOctetDeltaCount", |
| 24: "postPacketDeltaCount", |
| 25: "minimumIpTotalLength", |
| 26: "maximumIpTotalLength", |
| 27: "sourceIPv6Address", |
| 28: "destinationIPv6Address", |
| 29: "sourceIPv6PrefixLength", |
| 30: "destinationIPv6PrefixLength", |
| 31: "flowLabelIPv6", |
| 32: "icmpTypeCodeIPv4", |
| 33: "igmpType", |
| 34: "samplingInterval", |
| 35: "samplingAlgorithm", |
| 36: "flowActiveTimeout", |
| 37: "flowIdleTimeout", |
| 38: "engineType", |
| 39: "engineId", |
| 40: "exportedOctetTotalCount", |
| 41: "exportedMessageTotalCount", |
| 42: "exportedFlowRecordTotalCount", |
| 43: "ipv4RouterSc", |
| 44: "sourceIPv4Prefix", |
| 45: "destinationIPv4Prefix", |
| 46: "mplsTopLabelType", |
| 47: "mplsTopLabelIPv4Address", |
| 48: "samplerId", |
| 49: "samplerMode", |
| 50: "samplerRandomInterval", |
| 51: "classId", |
| 52: "minimumTTL", |
| 53: "maximumTTL", |
| 54: "fragmentIdentification", |
| 55: "postIpClassOfService", |
| 56: "sourceMacAddress", |
| 57: "postDestinationMacAddress", |
| 58: "vlanId", |
| 59: "postVlanId", |
| 60: "ipVersion", |
| 61: "flowDirection", |
| 62: "ipNextHopIPv6Address", |
| 63: "bgpNextHopIPv6Address", |
| 64: "ipv6ExtensionHeaders", |
| 70: "mplsTopLabelStackSection", |
| 71: "mplsLabelStackSection2", |
| 72: "mplsLabelStackSection3", |
| 73: "mplsLabelStackSection4", |
| 74: "mplsLabelStackSection5", |
| 75: "mplsLabelStackSection6", |
| 76: "mplsLabelStackSection7", |
| 77: "mplsLabelStackSection8", |
| 78: "mplsLabelStackSection9", |
| 79: "mplsLabelStackSection10", |
| 80: "destinationMacAddress", |
| 81: "postSourceMacAddress", |
| 82: "interfaceName", |
| 83: "interfaceDescription", |
| 84: "samplerName", |
| 85: "octetTotalCount", |
| 86: "packetTotalCount", |
| 87: "flagsAndSamplerId", |
| 88: "fragmentOffset", |
| 89: "forwardingStatus", |
| 90: "mplsVpnRouteDistinguisher", |
| 91: "mplsTopLabelPrefixLength", |
| 92: "srcTrafficIndex", |
| 93: "dstTrafficIndex", |
| 94: "applicationDescription", |
| 95: "applicationId", |
| 96: "applicationName", |
| 98: "postIpDiffServCodePoint", |
| 99: "multicastReplicationFactor", |
| 100: "className", |
| 101: "classificationEngineId", |
| 102: "layer2packetSectionOffset", |
| 103: "layer2packetSectionSize", |
| 104: "layer2packetSectionData", |
| 128: "bgpNextAdjacentAsNumber", |
| 129: "bgpPrevAdjacentAsNumber", |
| 130: "exporterIPv4Address", |
| 131: "exporterIPv6Address", |
| 132: "droppedOctetDeltaCount", |
| 133: "droppedPacketDeltaCount", |
| 134: "droppedOctetTotalCount", |
| 135: "droppedPacketTotalCount", |
| 136: "flowEndReason", |
| 137: "commonPropertiesId", |
| 138: "observationPointId", |
| 139: "icmpTypeCodeIPv6", |
| 140: "mplsTopLabelIPv6Address", |
| 141: "lineCardId", |
| 142: "portId", |
| 143: "meteringProcessId", |
| 144: "exportingProcessId", |
| 145: "templateId", |
| 146: "wlanChannelId", |
| 147: "wlanSSID", |
| 148: "flowId", |
| 149: "observationDomainId", |
| 150: "flowStartSeconds", |
| 151: "flowEndSeconds", |
| 152: "flowStartMilliseconds", |
| 153: "flowEndMilliseconds", |
| 154: "flowStartMicroseconds", |
| 155: "flowEndMicroseconds", |
| 156: "flowStartNanoseconds", |
| 157: "flowEndNanoseconds", |
| 158: "flowStartDeltaMicroseconds", |
| 159: "flowEndDeltaMicroseconds", |
| 160: "systemInitTimeMilliseconds", |
| 161: "flowDurationMilliseconds", |
| 162: "flowDurationMicroseconds", |
| 163: "observedFlowTotalCount", |
| 164: "ignoredPacketTotalCount", |
| 165: "ignoredOctetTotalCount", |
| 166: "notSentFlowTotalCount", |
| 167: "notSentPacketTotalCount", |
| 168: "notSentOctetTotalCount", |
| 169: "destinationIPv6Prefix", |
| 170: "sourceIPv6Prefix", |
| 171: "postOctetTotalCount", |
| 172: "postPacketTotalCount", |
| 173: "flowKeyIndicator", |
| 174: "postMCastPacketTotalCount", |
| 175: "postMCastOctetTotalCount", |
| 176: "icmpTypeIPv4", |
| 177: "icmpCodeIPv4", |
| 178: "icmpTypeIPv6", |
| 179: "icmpCodeIPv6", |
| 180: "udpSourcePort", |
| 181: "udpDestinationPort", |
| 182: "tcpSourcePort", |
| 183: "tcpDestinationPort", |
| 184: "tcpSequenceNumber", |
| 185: "tcpAcknowledgementNumber", |
| 186: "tcpWindowSize", |
| 187: "tcpUrgentPointer", |
| 188: "tcpHeaderLength", |
| 189: "ipHeaderLength", |
| 190: "totalLengthIPv4", |
| 191: "payloadLengthIPv6", |
| 192: "ipTTL", |
| 193: "nextHeaderIPv6", |
| 194: "mplsPayloadLength", |
| 195: "ipDiffServCodePoint", |
| 196: "ipPrecedence", |
| 197: "fragmentFlags", |
| 198: "octetDeltaSumOfSquares", |
| 199: "octetTotalSumOfSquares", |
| 200: "mplsTopLabelTTL", |
| 201: "mplsLabelStackLength", |
| 202: "mplsLabelStackDepth", |
| 203: "mplsTopLabelExp", |
| 204: "ipPayloadLength", |
| 205: "udpMessageLength", |
| 206: "isMulticast", |
| 207: "ipv4IHL", |
| 208: "ipv4Options", |
| 209: "tcpOptions", |
| 210: "paddingOctets", |
| 211: "collectorIPv4Address", |
| 212: "collectorIPv6Address", |
| 213: "exportInterface", |
| 214: "exportProtocolVersion", |
| 215: "exportTransportProtocol", |
| 216: "collectorTransportPort", |
| 217: "exporterTransportPort", |
| 218: "tcpSynTotalCount", |
| 219: "tcpFinTotalCount", |
| 220: "tcpRstTotalCount", |
| 221: "tcpPshTotalCount", |
| 222: "tcpAckTotalCount", |
| 223: "tcpUrgTotalCount", |
| 224: "ipTotalLength", |
| 225: "postNATSourceIPv4Address", |
| 226: "postNATDestinationIPv4Address", |
| 227: "postNAPTSourceTransportPort", |
| 228: "postNAPTDestinationTransportPort", |
| 229: "natOriginatingAddressRealm", |
| 230: "natEvent", |
| 231: "initiatorOctets", |
| 232: "responderOctets", |
| 233: "firewallEvent", |
| 234: "ingressVRFID", |
| 235: "egressVRFID", |
| 236: "VRFname", |
| 237: "postMplsTopLabelExp", |
| 238: "tcpWindowScale", |
| 239: "biflowDirection", |
| 240: "ethernetHeaderLength", |
| 241: "ethernetPayloadLength", |
| 242: "ethernetTotalLength", |
| 243: "dot1qVlanId", |
| 244: "dot1qPriority", |
| 245: "dot1qCustomerVlanId", |
| 246: "dot1qCustomerPriority", |
| 247: "metroEvcId", |
| 248: "metroEvcType", |
| 249: "pseudoWireId", |
| 250: "pseudoWireType", |
| 251: "pseudoWireControlWord", |
| 252: "ingressPhysicalInterface", |
| 253: "egressPhysicalInterface", |
| 254: "postDot1qVlanId", |
| 255: "postDot1qCustomerVlanId", |
| 256: "ethernetType", |
| 257: "postIpPrecedence", |
| 258: "collectionTimeMilliseconds", |
| 259: "exportSctpStreamId", |
| 260: "maxExportSeconds", |
| 261: "maxFlowEndSeconds", |
| 262: "messageMD5Checksum", |
| 263: "messageScope", |
| 264: "minExportSeconds", |
| 265: "minFlowStartSeconds", |
| 266: "opaqueOctets", |
| 267: "sessionScope", |
| 268: "maxFlowEndMicroseconds", |
| 269: "maxFlowEndMilliseconds", |
| 270: "maxFlowEndNanoseconds", |
| 271: "minFlowStartMicroseconds", |
| 272: "minFlowStartMilliseconds", |
| 273: "minFlowStartNanoseconds", |
| 274: "collectorCertificate", |
| 275: "exporterCertificate", |
| 276: "dataRecordsReliability", |
| 277: "observationPointType", |
| 278: "newConnectionDeltaCount", |
| 279: "connectionSumDurationSeconds", |
| 280: "connectionTransactionId", |
| 281: "postNATSourceIPv6Address", |
| 282: "postNATDestinationIPv6Address", |
| 283: "natPoolId", |
| 284: "natPoolName", |
| 285: "anonymizationFlags", |
| 286: "anonymizationTechnique", |
| 287: "informationElementIndex", |
| 288: "p2pTechnology", |
| 289: "tunnelTechnology", |
| 290: "encryptedTechnology", |
| 291: "basicList", |
| 292: "subTemplateList", |
| 293: "subTemplateMultiList", |
| 294: "bgpValidityState", |
| 295: "IPSecSPI", |
| 296: "greKey", |
| 297: "natType", |
| 298: "initiatorPackets", |
| 299: "responderPackets", |
| 300: "observationDomainName", |
| 301: "selectionSequenceId", |
| 302: "selectorId", |
| 303: "informationElementId", |
| 304: "selectorAlgorithm", |
| 305: "samplingPacketInterval", |
| 306: "samplingPacketSpace", |
| 307: "samplingTimeInterval", |
| 308: "samplingTimeSpace", |
| 309: "samplingSize", |
| 310: "samplingPopulation", |
| 311: "samplingProbability", |
| 312: "dataLinkFrameSize", |
| 313: "ipHeaderPacketSection", |
| 314: "ipPayloadPacketSection", |
| 315: "dataLinkFrameSection", |
| 316: "mplsLabelStackSection", |
| 317: "mplsPayloadPacketSection", |
| 318: "selectorIdTotalPktsObserved", |
| 319: "selectorIdTotalPktsSelected", |
| 320: "absoluteError", |
| 321: "relativeError", |
| 322: "observationTimeSeconds", |
| 323: "observationTimeMilliseconds", |
| 324: "observationTimeMicroseconds", |
| 325: "observationTimeNanoseconds", |
| 326: "digestHashValue", |
| 327: "hashIPPayloadOffset", |
| 328: "hashIPPayloadSize", |
| 329: "hashOutputRangeMin", |
| 330: "hashOutputRangeMax", |
| 331: "hashSelectedRangeMin", |
| 332: "hashSelectedRangeMax", |
| 333: "hashDigestOutput", |
| 334: "hashInitialiserValue", |
| 335: "selectorName", |
| 336: "upperCILimit", |
| 337: "lowerCILimit", |
| 338: "confidenceLevel", |
| 339: "informationElementDataType", |
| 340: "informationElementDescription", |
| 341: "informationElementName", |
| 342: "informationElementRangeBegin", |
| 343: "informationElementRangeEnd", |
| 344: "informationElementSemantics", |
| 345: "informationElementUnits", |
| 346: "privateEnterpriseNumber", |
| 347: "virtualStationInterfaceId", |
| 348: "virtualStationInterfaceName", |
| 349: "virtualStationUUID", |
| 350: "virtualStationName", |
| 351: "layer2SegmentId", |
| 352: "layer2OctetDeltaCount", |
| 353: "layer2OctetTotalCount", |
| 354: "ingressUnicastPacketTotalCount", |
| 355: "ingressMulticastPacketTotalCount", |
| 356: "ingressBroadcastPacketTotalCount", |
| 357: "egressUnicastPacketTotalCount", |
| 358: "egressBroadcastPacketTotalCount", |
| 359: "monitoringIntervalStartMilliSeconds", |
| 360: "monitoringIntervalEndMilliSeconds", |
| 361: "portRangeStart", |
| 362: "portRangeEnd", |
| 363: "portRangeStepSize", |
| 364: "portRangeNumPorts", |
| 365: "staMacAddress", |
| 366: "staIPv4Address", |
| 367: "wtpMacAddress", |
| 368: "ingressInterfaceType", |
| 369: "egressInterfaceType", |
| 370: "rtpSequenceNumber", |
| 371: "userName", |
| 372: "applicationCategoryName", |
| 373: "applicationSubCategoryName", |
| 374: "applicationGroupName", |
| 375: "originalFlowsPresent", |
| 376: "originalFlowsInitiated", |
| 377: "originalFlowsCompleted", |
| 378: "distinctCountOfSourceIPAddress", |
| 379: "distinctCountOfDestinationIPAddress", |
| 380: "distinctCountOfSourceIPv4Address", |
| 381: "distinctCountOfDestinationIPv4Address", |
| 382: "distinctCountOfSourceIPv6Address", |
| 383: "distinctCountOfDestinationIPv6Address", |
| 384: "valueDistributionMethod", |
| 385: "rfc3550JitterMilliseconds", |
| 386: "rfc3550JitterMicroseconds", |
| 387: "rfc3550JitterNanoseconds", |
| 388: "dot1qDEI", |
| 389: "dot1qCustomerDEI", |
| 390: "flowSelectorAlgorithm", |
| 391: "flowSelectedOctetDeltaCount", |
| 392: "flowSelectedPacketDeltaCount", |
| 393: "flowSelectedFlowDeltaCount", |
| 394: "selectorIDTotalFlowsObserved", |
| 395: "selectorIDTotalFlowsSelected", |
| 396: "samplingFlowInterval", |
| 397: "samplingFlowSpacing", |
| 398: "flowSamplingTimeInterval", |
| 399: "flowSamplingTimeSpacing", |
| 400: "hashFlowDomain", |
| 401: "transportOctetDeltaCount", |
| 402: "transportPacketDeltaCount", |
| 403: "originalExporterIPv4Address", |
| 404: "originalExporterIPv6Address", |
| 405: "originalObservationDomainId", |
| 406: "intermediateProcessId", |
| 407: "ignoredDataRecordTotalCount", |
| 408: "dataLinkFrameType", |
| 409: "sectionOffset", |
| 410: "sectionExportedOctets", |
| 411: "dot1qServiceInstanceTag", |
| 412: "dot1qServiceInstanceId", |
| 413: "dot1qServiceInstancePriority", |
| 414: "dot1qCustomerSourceMacAddress", |
| 415: "dot1qCustomerDestinationMacAddress", |
| 417: "postLayer2OctetDeltaCount", |
| 418: "postMCastLayer2OctetDeltaCount", |
| 420: "postLayer2OctetTotalCount", |
| 421: "postMCastLayer2OctetTotalCount", |
| 422: "minimumLayer2TotalLength", |
| 423: "maximumLayer2TotalLength", |
| 424: "droppedLayer2OctetDeltaCount", |
| 425: "droppedLayer2OctetTotalCount", |
| 426: "ignoredLayer2OctetTotalCount", |
| 427: "notSentLayer2OctetTotalCount", |
| 428: "layer2OctetDeltaSumOfSquares", |
| 429: "layer2OctetTotalSumOfSquares", |
| 430: "layer2FrameDeltaCount", |
| 431: "layer2FrameTotalCount", |
| 432: "pseudoWireDestinationIPv4Address", |
| 433: "ignoredLayer2FrameTotalCount", |
| 434: "mibObjectValueInteger", |
| 435: "mibObjectValueOctetString", |
| 436: "mibObjectValueOID", |
| 437: "mibObjectValueBits", |
| 438: "mibObjectValueIPAddress", |
| 439: "mibObjectValueCounter", |
| 440: "mibObjectValueGauge", |
| 441: "mibObjectValueTimeTicks", |
| 442: "mibObjectValueUnsigned", |
| 443: "mibObjectValueTable", |
| 444: "mibObjectValueRow", |
| 445: "mibObjectIdentifier", |
| 446: "mibSubIdentifier", |
| 447: "mibIndexIndicator", |
| 448: "mibCaptureTimeSemantics", |
| 449: "mibContextEngineID", |
| 450: "mibContextName", |
| 451: "mibObjectName", |
| 452: "mibObjectDescription", |
| 453: "mibObjectSyntax", |
| 454: "mibModuleName", |
| 455: "mobileIMSI", |
| 456: "mobileMSISDN", |
| 457: "httpStatusCode", |
| 458: "sourceTransportPortsLimit", |
| 459: "httpRequestMethod", |
| 460: "httpRequestHost", |
| 461: "httpRequestTarget", |
| 462: "httpMessageVersion", |
| 466: "natQuotaExceededEvent", |
| 471: "maxSessionEntries", |
| 472: "maxBIBEntries", |
| 473: "maxEntriesPerUser", |
| 475: "maxFragmentsPendingReassembly", |
| } |
| |
| |
| class IPFIX(Packet): |
| name = "IPFIX" |
| fields_desc = [ |
| ShortField("version", 10), |
| ShortField("length", None), |
| IntField("exportTime", None), |
| IntField("sequenceNumber", 1), |
| IntField("observationDomainID", 1), |
| ] |
| |
| |
| class FieldSpecifier(Packet): |
| name = "Field Specifier" |
| fields_desc = [ |
| ShortEnumField("informationElement", None, information_elements), |
| ShortField("fieldLength", None), |
| ] |
| |
| def extract_padding(self, s): |
| return "", s |
| |
| |
| class Template(Packet): |
| name = "Template" |
| fields_desc = [ |
| ShortField("templateID", 256), |
| FieldLenField("fieldCount", None, count_of="fields"), |
| PacketListField( |
| "templateFields", [], FieldSpecifier, count_from=lambda p: p.fieldCount |
| ), |
| ] |
| |
| |
| class Data(Packet): |
| name = "Data" |
| fields_desc = [ |
| StrLenField("data", "", length_from=lambda p: p.underlayer.length - 4) |
| ] |
| |
| def extract_padding(self, s): |
| return "", s |
| |
| |
| class Set(Packet): |
| name = "Set" |
| fields_desc = [ShortField("setID", 256), ShortField("length", None)] |
| |
| def guess_payload_class(self, payload): |
| if self.setID == 2: |
| return Template |
| elif self.setID > 255: |
| return Data |
| else: |
| return Packet.guess_payload_class(self, payload) |
| |
| |
| bind_layers(IPFIX, Set) |
| bind_layers(UDP, IPFIX, dport=4739) |
| |
| |
| class IPFIXDecoder: |
| """IPFIX data set decoder""" |
| |
| def __init__(self): |
| self._templates = [] |
| |
| def add_template(self, template): |
| """ |
| Add IPFIX template |
| |
| :param template: IPFIX template |
| """ |
| templateID = template.templateID |
| fields = [] |
| rec_len = 0 |
| for field in template.templateFields: |
| fields.append({"name": field.informationElement, "len": field.fieldLength}) |
| rec_len += field.fieldLength |
| self._templates.append({"id": templateID, "fields": fields, "rec_len": rec_len}) |
| |
| def decode_data_set(self, data_set): |
| """ |
| Decode IPFIX data |
| |
| :param data_set: IPFIX data set |
| :returns: List of decoded data records. |
| """ |
| data = [] |
| for template in self._templates: |
| if template["id"] == data_set.setID: |
| offset = 0 |
| d = data_set[Data].data |
| for i in range(len(d) // template["rec_len"]): |
| record = {} |
| for field in template["fields"]: |
| f = d[offset : offset + field["len"]] |
| offset += field["len"] |
| record.update({field["name"]: f}) |
| data.append(record) |
| break |
| return data |