Netdump library

libraries/Netdump/README.md

@@ -0,0 +1,52 @@
+
+esp8266/Arduino goodies
+-----------------------
+
+* NetDump (lwip2)  
+  Packet sniffer library to help study network issues, check example-sketches  
+  Log examples on serial console:
+```
+14:07:01.854 ->  in 0  ARP who has 10.43.1.117 tell 10.43.1.254
+14:07:01.854 -> out 0  ARP 10.43.1.117 is at 5c:cf:7f:c3:ad:51
+
+[...] hello-world, dumped in packets:
+14:07:46.227 ->  in 0  IPv4 10.43.1.254>10.43.1.117 TCP 54546>2[P.] seq:1945448681..1945448699 ack:6618 win:29200 len=18
+14:07:46.260 -> 5c cf 7f c3 ad 51 74 da 38 3a 1f 61 08 00 45 10 \..Qt.8:.a..E.
+14:07:46.260 -> 00 3a b2 bc 40 00 40 06 70 29 0a 2b 01 fe 0a 2b .:..@.@.p).+...+
+14:07:46.260 -> 01 75 d5 12 00 02 73 f5 30 e9 00 00 19 da 50 18 .u....s.0.....P.
+14:07:46.260 -> 72 10 f8 da 00 00 70 6c 20 68 65 6c 6c 6f 2d 77 r.....pl hello-w
+14:07:46.260 -> 6f 72 6c 64 20 31 0d 0a                         orld 1..
+14:07:46.294 -> out 0  IPv4 10.43.1.117>10.43.1.254 TCP 2>54546[P.] seq:6618..6619 ack:1945448699 win:2126 len=1
+14:07:46.326 -> 00 20 00 00 00 00 aa aa 03 00 00 00 08 00 45 00 . ............E.
+14:07:46.326 -> 00 29 00 0d 00 00 ff 06 a3 f9 0a 2b 01 75 0a 2b .).........+.u.+
+14:07:46.327 -> 01 fe 00 02 d5 12 00 00 19 da 73 f5 30 fb 50 18 ..........s.0.P.
+14:07:46.327 -> 08 4e 93 d5 00 00 68                            .N....h
+14:07:46.327 ->  in 0  IPv4 10.43.1.254>10.43.1.117 TCP 54546>2[.] seq:1945448699 ack:6619 win:29200
+14:07:46.327 -> 5c cf 7f c3 ad 51 74 da 38 3a 1f 61 08 00 45 10 \..Qt.8:.a..E.
+14:07:46.360 -> 00 28 b2 bd 40 00 40 06 70 3a 0a 2b 01 fe 0a 2b .(..@.@.p:.+...+
+14:07:46.360 -> 01 75 d5 12 00 02 73 f5 30 fb 00 00 19 db 50 10 .u....s.0.....P.
+14:07:46.360 -> 72 10 92 1b 00 00                               r.....
+14:07:46.360 -> out 0  IPv4 10.43.1.117>10.43.1.254 TCP 2>54546[P.] seq:6619..6630 ack:1945448699 win:2126 len=11
+14:07:46.360 -> 00 20 00 00 00 00 aa aa 03 00 00 00 08 00 45 00 . ............E.
+14:07:46.360 -> 00 33 00 0e 00 00 ff 06 a3 ee 0a 2b 01 75 0a 2b .3.........+.u.+
+14:07:46.393 -> 01 fe 00 02 d5 12 00 00 19 db 73 f5 30 fb 50 18 ..........s.0.P.
+14:07:46.393 -> 08 4e 16 a1 00 00 65 6c 6c 6f 2d 77 6f 72 6c 64 .N....ello-world
+14:07:46.393 -> 0a                                              .
+
+[...] help protocol decoding from inside the esp
+14:08:11.715 ->  in 0  IPv4 10.43.1.254>239.255.255.250 UDP 50315>1900 len=172
+14:08:11.716 -> 01 00 5e 7f ff fa 74 da 38 3a 1f 61 08 00 45 00 ....t.8:.a..E.
+14:08:11.716 -> 00 c8 9b 40 40 00 01 11 e1 c1 0a 2b 01 fe ef ff ...@@......+....
+14:08:11.749 -> ff fa c4 8b 07 6c 00 b4 9c 28 4d 2d 53 45 41 52 .....l...(M-SEAR
+14:08:11.749 -> 43 48 20 2a 20 48 54 54 50 2f 31 2e 31 0d 0a 48 CH * HTTP/1.1..H
+14:08:11.749 -> 4f 53 54 3a 20 32 33 39 2e 32 35 35 2e 32 35 35 OST: 239.255.255
+14:08:11.749 -> 2e 32 35 30 3a 31 39 30 30 0d 0a 4d 41 4e 3a 20 .250:1900..MAN: 
+14:08:11.749 -> 22 73 73 64 70 3a 64 69 73 63 6f 76 65 72 22 0d "ssdp:discover".
+14:08:11.749 -> 0a 4d 58 3a 20 31 0d 0a 53 54 3a 20 75 72 6e 3a .MX: 1..ST: urn:
+14:08:11.782 -> 64 69 61 6c 2d 6d 75 6c 74 69 73 63 72 65 65 6e dial-multiscreen
+14:08:11.782 -> 2d 6f 72 67 3a 73 65 72 76 69 63 65 3a 64 69 61 -org:service:dia
+14:08:11.782 -> 6c 3a 31 0d 0a 55 53 45 52 2d 41 47 45 4e 54 3a l:1..USER-AGENT:
+14:08:11.782 -> 20 47 6f 6f 67 6c 65 20 43 68 72 6f 6d 65 2f 36  Google Chrome/6
+14:08:11.782 -> 36 2e 30 2e 33 33 35 39 2e 31 31 37 20 4c 69 6e 6.0.3359.117 Lin
+14:08:11.782 -> 75 78 0d 0a 0d 0a                               ux....
+

libraries/Netdump/astyle_goodies.conf

@@ -0,0 +1,32 @@
+# Code formatting rules for Arduino examples, taken from:
+#
+#   https://github.com/arduino/Arduino/blob/master/build/shared/examples_formatter.conf
+#
+
+mode=c
+lineend=linux
+style=allman
+
+# 4 spaces indentation
+indent=spaces=4
+
+# also indent macros
+#indent-preprocessor
+
+# indent classes, switches (and cases), comments starting at column 1
+indent-col1-comments
+
+# put a space around operators
+pad-oper
+
+# put a space after if/for/while
+pad-header
+
+# if you like one-liners, keep them
+keep-one-line-statements
+
+attach-closing-while
+unpad-paren
+pad-oper
+remove-comment-prefix
+add-braces

libraries/Netdump/examples/Netdump.ino

@@ -0,0 +1,75 @@
+// Do not remove the include below
+#include "NetDumpTest.h"
+#include <ESP8266WiFi.h>
+#include "LocalDefines.h"
+#include <ESP8266WebServer.h>
+#include <Netdump.h>
+#include <FS.h>
+
+/*
+  dump network packets on serial console
+  released to the public domain
+*/
+
+
+Netdump nd;
+ESP8266WebServer server(80);
+WiFiServer ws(8000);
+
+
+void handleRoot() {
+  static int rq = 0;
+  String a = "<h1>You are connected, rq = "+String(rq++)+"</h1>";
+  server.send(200, "text/html", a);
+}
+
+
+File outfile = SPIFFS.open("test", "w");
+
+void setup(void) {
+  Serial.begin(115200);
+
+  WiFi.mode(WIFI_STA);
+  WiFi.begin(ssid,password);
+  SPIFFS.begin();
+
+  server.on("/", handleRoot);
+  server.begin();
+
+/*
+// To serial, include hex print only localIP traffic
+  nd.printDump(Serial, true,
+		  	  	  [](NetdumpPacket n)
+				  {
+	  	  	  	  	  return ((n.ethType() != 0x8912) &&
+	  	  	  	  			  ((n.sourceIP()==WiFi.localIP()) || n.destIP()==WiFi.localIP()) );
+				  });
+*/
+
+/*
+// To serial, include hex print only localIP traffic
+  nd.printDump(Serial, false); // To serial all traffic, no hex dump
+*/
+
+/*
+    nd.printDump(outfile, false); // To file all traffic, no hex dump, format ascii
+*/
+
+/*
+    nd.fileDump(outfile, false); // To file all traffic, no hex dump, format pcap file
+*/
+
+/*
+// To telnet, all traffic, use `n 10.0.0.212 8000 | tcpdump -r -`
+  ws.begin();
+  nd.tcpDump(ws);
+
+*/
+
+
+}
+
+void loop(void) {
+	  server.handleClient();
+
+}

libraries/Netdump/keywords.txt

@@ -0,0 +1 @@
+

libraries/Netdump/library.properties

@@ -0,0 +1,9 @@
+name=NetDump
+version=2
+author=David Gauchard
+maintainer=David Gauchard
+sentence=tcpdump-like logger for esp8266/Arduino
+paragraph=Dumps input / output packets on "Print"able type, or provide a TCP server for the real tcpdump. Check examples. Some other unrelated and independant tools are included.
+category=Communication
+url=https://
+architectures=esp8266

libraries/Netdump/src/Netdump.cpp

@@ -0,0 +1,210 @@
+/*
+    NetDump library - tcpdump-like packet logger facility
+
+    Copyright (c) 2019 Herman Reintke. All rights reserved.
+    This file is part of the esp8266 core for Arduino environment.
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+*/
+
+#include "Netdump.h"
+#include <lwip/init.h>
+#include "Schedule.h"
+
+Netdump* Netdump::self;
+
+void Netdump::setCallback(NetdumpCallback nc)
+{
+    netDumpCallback = nc;
+}
+void Netdump::setCallback(NetdumpCallback nc, NetdumpFilter nf)
+{
+    netDumpFilter   = nf;
+    netDumpCallback = nc;
+}
+void Netdump::setFilter(NetdumpFilter nf)
+{
+    netDumpFilter = nf;
+}
+void Netdump::reset()
+{
+    setCallback(nullptr, nullptr);
+}
+void Netdump::printDump(Print& out, NetdumpPacket::PacketDetail ndd, NetdumpFilter nf)
+{
+    out.printf("netDump starting\r\n");
+//    setCallback(std::bind(&Netdump::printDumpProcess, this, std::ref(out), ndd, std::placeholders::_1), nf);
+    setCallback([&out, ndd, this](NetdumpPacket & ndp)
+    {
+        printDumpProcess(out, ndd, ndp);
+    }, nf);
+
+
+}
+void Netdump::fileDump(File outfile, NetdumpFilter nf)
+{
+
+    //char buf[24];
+
+    uint32_t buf[6];
+/*
+    *(uint32_t*)&buf[0] = 0xa1b2c3d4;
+    *(uint32_t*)&buf[4] = 0x00040002;
+    *(uint32_t*)&buf[8] = 0;
+    *(uint32_t*)&buf[12] = 0;
+    *(uint32_t*)&buf[16] = 1024;
+    *(uint32_t*)&buf[20] = 1;
+*/
+    buf[0] = 0xa1b2c3d4;
+    buf[1] = 0x00040002;
+    buf[2] = 0;
+    buf[3] = 0;
+    buf[4] = 1024;
+    buf[5] = 1;
+
+    outfile.write((uint8_t*)buf, 24);
+    //	setCallback( std::bind(&Netdump::fileDumpProcess, this, outfile, std::placeholders::_1));
+    setCallback([outfile, this](NetdumpPacket & ndp)
+    {
+        fileDumpProcess(outfile, ndp);
+    }, nf);
+}
+void Netdump::tcpDump(WiFiServer &tcpDumpServer, NetdumpFilter nf)
+{
+    // Get initialize code from netdumpout.cpp
+    if (packetBuffer)
+    {
+        delete packetBuffer;
+    }
+    packetBuffer = new char[2048];
+    bufferIndex = 0;
+
+    //	schedule_function(std::bind(&Netdump::tcpDumpLoop,this,std::ref(tcpDumpServer)));
+    schedule_function([&tcpDumpServer, this]()
+    {
+        tcpDumpLoop(tcpDumpServer);
+    });
+    Serial.printf("scheduled\r\n");
+}
+
+void Netdump::capture(int netif_idx, const char* data, size_t len, int out, int success)
+{
+    NetdumpPacket np(netif_idx, data, len, out, success);
+    if (self->netDumpCallback)
+    {
+        if (self->netDumpFilter  && !self->netDumpFilter(np))
+        {
+            return;
+        }
+        self->netDumpCallback(np);
+    }
+}
+
+void Netdump::printDumpProcess(Print& out, NetdumpPacket::PacketDetail ndd, const NetdumpPacket& np)
+{
+    out.printf("%8d %s", millis(), np.toString(ndd).c_str());
+}
+
+void Netdump::fileDumpProcess(File outfile, const NetdumpPacket& np)
+{
+    size_t incl_len = np.len > 1024 ? 1024 : np.len;
+    char buf[16];
+
+    struct timeval tv;
+    gettimeofday(&tv, nullptr);
+    *(uint32_t*)&buf[0] = tv.tv_sec;
+    *(uint32_t*)&buf[4] = tv.tv_usec;
+    *(uint32_t*)&buf[8] = incl_len;
+    *(uint32_t*)&buf[12] = np.len;
+    outfile.write(buf, 16);
+
+    outfile.write(np.data, incl_len);
+}
+void Netdump::tcpDumpProcess(const NetdumpPacket& np)
+{
+    // Get capture code from netdumpout.cpp
+    if (np.isIPv4() && np.isTCP()
+            && ((np.out && np.getSrcPort() == tcpDumpClient.localPort())
+                || (!np.out && np.getDstPort() == tcpDumpClient.localPort())
+               )
+       )
+    {
+        // skip myself
+        return;
+    }
+    size_t incl_len = np.len > 1024 ? 1024 : np.len;
+
+    struct timeval tv;
+    gettimeofday(&tv, nullptr);
+    *(uint32_t*)&packetBuffer[bufferIndex] = tv.tv_sec;
+    *(uint32_t*)&packetBuffer[bufferIndex + 4] = tv.tv_usec;
+    *(uint32_t*)&packetBuffer[bufferIndex + 8] = incl_len;
+    *(uint32_t*)&packetBuffer[bufferIndex + 12] = np.len;
+    bufferIndex += 16;
+    memcpy(&packetBuffer[bufferIndex], np.data, incl_len);
+    bufferIndex += incl_len;
+    if (bufferIndex && tcpDumpClient && tcpDumpClient.availableForWrite() >= bufferIndex)
+    {
+        tcpDumpClient.write(packetBuffer, bufferIndex);
+        bufferIndex = 0;
+    }
+
+
+}
+void Netdump::tcpDumpLoop(WiFiServer &tcpDumpServer)
+{
+    if (tcpDumpServer.hasClient())
+    {
+        tcpDumpClient = tcpDumpServer.available();
+        //if (fastsend)
+        tcpDumpClient.setNoDelay(true);
+
+
+        // pcap-savefile(5) capture preamble
+        *(uint32_t*)&packetBuffer[0] = 0xa1b2c3d4;
+        *(uint32_t*)&packetBuffer[4] = 0x00040002;
+        *(uint32_t*)&packetBuffer[8] = 0;
+        *(uint32_t*)&packetBuffer[12] = 0;
+        *(uint32_t*)&packetBuffer[16] = 1024;
+        *(uint32_t*)&packetBuffer[20] = 1;
+        tcpDumpClient.write(packetBuffer, 24);
+        bufferIndex = 0;
+        //      setCallback(std::bind(&Netdump::tcpDumpProcess,this,std::placeholders::_1));
+        setCallback([this](NetdumpPacket & ndp)
+        {
+            tcpDumpProcess(ndp);
+        });
+
+        Serial.printf("client started\r\n");
+    }
+    if (!tcpDumpClient || !tcpDumpClient.connected())
+    {
+        setCallback(nullptr);
+    }
+    if (bufferIndex && tcpDumpClient && tcpDumpClient.availableForWrite() >= bufferIndex)
+    {
+        Serial.printf("tcp write %d\r\n", bufferIndex);
+        tcpDumpClient.write(packetBuffer, bufferIndex);
+        bufferIndex = 0;
+    }
+    //  schedule_function(std::bind(&Netdump::tcpDumpLoop,this,std::ref(tcpDumpServer)));
+    if (tcpDumpServer.status() != CLOSED)
+    {
+        schedule_function([&tcpDumpServer, this]()
+        {
+            tcpDumpLoop(tcpDumpServer);
+        });
+    }
+}