diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 11d6b37..bd3cb86 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -10,8 +10,26 @@ jobs: checks: runs-on: ubuntu-latest steps: - - name: Dependencies - run: sudo apt install libpcap-dev iptables + - name: Basic Dependencies + run: | + sudo apt-get update + sudo apt-get install -y libpcap-dev iptables zlib1g-dev build-essential + + - name: Install spicyc + run: | + wget https://github.com/zeek/spicy/releases/download/v1.13.1/spicy_linux_ubuntu24.deb + sudo dpkg --install spicy_linux_ubuntu24.deb + sudo apt-get install -f -y # pulling in any missing deps + rm spicy_linux_ubuntu24.deb + + - name: Add Spicy CLI to PATH + run: echo "/opt/spicy/bin" >> $GITHUB_PATH + + - name: Install clang17 + run: | + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 17 - name: Checkout uses: actions/checkout@v3 @@ -19,10 +37,20 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: "^1.21" + go-version: "^1.23" + + - name: Build Spicy generated files + run: | + make spicy - name: Build + env: + CC: clang + CXX: clang++ run: go build -v ./... - name: Test + env: + CC: clang + CXX: clang++ run: go test -v ./... diff --git a/.gitignore b/.gitignore index a3f4a9b..6837cac 100644 --- a/.gitignore +++ b/.gitignore @@ -37,10 +37,12 @@ poc/ # Dev .vscode -openspec/ -.cache -.codex -openspec/ -docs/roadmap.md +.cache/ +.codex/ docs/engineering-guidelines.md +docs/roadmap.md +openspec/ +# Spicy generated files +protocols/spicy/*.cc +protocols/spicy/parsers/*.h diff --git a/Makefile b/Makefile index eedd1d5..990178b 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,10 @@ upx: default: build build: - go build -ldflags=$(LDFLAGS) -o bin/server app/server.go + CC=clang CXX=clang++ go build -ldflags=$(LDFLAGS) -o bin/server app/server.go + +spicy: + cd protocols/spicy && make static: go build --ldflags '-extldflags "-static"' -o bin/server app/server.go diff --git a/config/config.yaml b/config/config.yaml index 69c64ee..f95053a 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -27,4 +27,7 @@ max_tcp_payload: 4096 # bytes dial_timeout: 5 # timeout in seconds for proxy target connection. capture_traffic: - enabled: false \ No newline at end of file + enabled: false + +spicy: + enabled: true diff --git a/docs/configuration.md b/docs/configuration.md index c5d711c..03e986d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -17,6 +17,7 @@ This file holds the core settings for Glutton. Key configuration options include - **max_tcp_payload:** Maximum TCP payload size in bytes (default: `4096`). Proxy TCP uses this as the per-direction captured payload cap. - **dial_timeout:** Timeout, in seconds, for opening outbound proxy TCP target connections (default: `5`). - **capture_traffic.enabled:** Enables raw payload capture in logs and produced decoded events. When disabled, proxy TCP still forwards traffic and logs metadata, but raw payload bytes are omitted from decoded events. +- **spicy.enabled:** Enables Spicy parser integration for supported protocols. When enabled, Glutton initializes the Spicy/HILTI runtime and uses Spicy-backed parsing paths where implemented. When disabled, Glutton uses the existing Go parser and TCP fallback paths. - **confpath:** The directory path where the configuration file resides. - **producers:** - **enabled**: Boolean flag to enable or disable logging/producer functionality. @@ -61,6 +62,9 @@ dial_timeout: 5 capture_traffic: enabled: false + +spicy: + enabled: true ``` ### config/rules.yaml diff --git a/docs/extension.md b/docs/extension.md index f1963d9..906bb9d 100644 --- a/docs/extension.md +++ b/docs/extension.md @@ -58,8 +58,57 @@ For example: - Write tests similar to those in `protocols/protocols_test.go` to verify your new handler’s functionality. - Use `go test` to ensure that your changes do not break existing functionality. +## Spicy Parser Integration + +Glutton can use [Spicy](https://docs.zeek.org/projects/spicy/en/latest/) parsers when `spicy.enabled: true` is set in `config/config.yaml`. Spicy parsing is currently used for: + +- HTTP request parsing through `HTTP::Request`, with Glutton behavior implemented in `protocols/spicy/handlers/http.go`. +- TCP payload protocol detection through `TCP::Protocol`, which helps route raw TCP traffic to HTTP, RDP, MongoDB, or the fallback TCP handler. + +Spicy does not replace Glutton's handler layer. The parser extracts structured fields from bytes; Go handlers still own connection handling, logging, producer events, responses, and fallback behavior. + +### Architecture + +The Spicy integration has three main parts: + +- **C++ bridge:** `protocols/spicy/bridge.{h,cpp}` initializes and shuts down the Spicy/HILTI runtime, exposes the generic `spicy_parse_generic()` entry point, lists compiled parsers, and flattens parsed HILTI values into key-value fields. +- **Generated parser code:** `make spicy` compiles `protocols/spicy/parsers/*.spicy` into generated C++ files and creates the combined Spicy linker file used to register parser modules. +- **Go parser worker:** `protocols/spicy/parser.go` runs Spicy calls on a dedicated OS thread with `runtime.LockOSThread()`, communicates through a command channel, keeps the parser registry, and applies timeout protection around parsing. + +The bridge is intentionally protocol-agnostic. New parsers should not add parser-specific includes or parser-specific logic to `bridge.cpp`; concrete parser modules are registered through the generated linker code. + +### Parsed Field Shape + +The bridge recursively flattens Spicy/HILTI values into a flat map: + +- Struct fields become `field` or `parent.field`. +- Vectors and arrays become `field[0]`, `field[1]`, and so on. +- Maps become `field.key`. +- Nested values are depth-limited to prevent runaway recursion. + +For example, an HTTP parser can expose fields such as `method`, `path`, `query`, `version.number`, or `headers[0].name`. + +### TCP Protocol Detection + +When Spicy is enabled, the raw TCP dispatcher can call `spicy.Parse("tcp", sample)` before falling back to the older byte-prefix checks. The `TCP::Protocol` parser inspects application payload bytes, not TCP/IP headers. + +The detector currently recognizes: + +- HTTP from the first request bytes. +- RDP from the initial connection bytes. +- MongoDB from a valid 16-byte message header. + +Unknown or invalid samples continue through the normal fallback TCP handler. + +### Adding a Spicy Parser + +1. Add a grammar under `protocols/spicy/parsers/*.spicy`. +2. Run `make spicy` before building or testing so generated parser C++ files, parser headers, and the combined linker file are available. +3. Use the parser from Go through `spicy.Parse("", payload)`. Parser names are registered from compiled Spicy module names, for example `HTTP::Request` is available as `http`. +4. Add or update Glutton routing and handler code when the parser should be used for live traffic. A compiled grammar makes parsing available, but it does not decide how much data to read, what handler to call, what response to send, or which producer event shape to emit. +5. Add tests for the parser and any handler or routing behavior that consumes it. + ## Customizing Logging and Rules - **Logging:** The logging mechanism is provided by the Producer (located in `producer/`). You can modify or extend it to suit your logging infrastructure. - **Rules Engine:** The rules engine (found in `rules/`) can be extended to support additional matching criteria or custom rule types. - diff --git a/docs/setup.md b/docs/setup.md index 879d67e..1bd694b 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -20,9 +20,11 @@ Ensure you have [Go](https://go.dev/dl/) installed (recommended version: **Go 1. ```bash sudo apt-get update -sudo apt-get install gcc libpcap-dev iptables +sudo apt-get install gcc g++ libpcap-dev iptables ``` +Spicy parser development also requires Spicy/HILTI headers and libraries under `/opt/spicy` and a C++20-capable compiler. After installing Spicy, run `make spicy` before `make build` or `go test ./...`. The `make spicy` target generates parser C++ files, parser headers, and the combined linker file used by the generic Spicy bridge. + ### Arch Linux ```bash sudo pacman -S gcc libpcap iptables @@ -30,7 +32,7 @@ sudo pacman -S gcc libpcap iptables ### Fedora ```bash -sudo dnf install gcc libpcap-devel iptables +sudo dnf install gcc gcc-c++ libpcap-devel iptables ``` ## Building Glutton @@ -103,4 +105,3 @@ To deploy using Docker: ``` The Docker container is preconfigured with the necessary dependencies (iptables, libpcap, etc.) and copies the configuration and rules files into the container. - diff --git a/glutton.go b/glutton.go index 74edea3..8c03375 100644 --- a/glutton.go +++ b/glutton.go @@ -18,6 +18,7 @@ import ( "github.com/mushorg/glutton/connection" "github.com/mushorg/glutton/producer" "github.com/mushorg/glutton/protocols" + "github.com/mushorg/glutton/protocols/spicy" "github.com/mushorg/glutton/rules" "github.com/google/uuid" @@ -134,6 +135,13 @@ func (g *Glutton) Init() error { g.tcpProtocolHandlers = protocols.MapTCPProtocolHandlers(g.Logger, g) g.udpProtocolHandlers = protocols.MapUDPProtocolHandlers(g.Logger, g) + // Initializing Spicy parsers + if viper.GetBool("spicy.enabled") { + if err := spicy.Initialize(g.Logger); err != nil { + return fmt.Errorf("failed to initialize Spicy: %w", err) + } + } + return nil } @@ -366,7 +374,12 @@ func (g *Glutton) Shutdown() { if err := flushTProxyIPTables(viper.GetString("interface"), g.publicAddrs[0].String(), "udp", uint32(g.Server.udpPort), uint32(viper.GetInt("ports.ssh"))); err != nil { g.Logger.Error("Failed to drop udp iptables", producer.ErrAttr(err)) } - + if viper.GetBool("spicy.enabled") { + g.Logger.Info("Cleaning up and shutting down Spicy and HILTI runtimes") + if err := spicy.Cleanup(); err != nil { + g.Logger.Error("Failed to clean up Spicy and HILTI runtimes", producer.ErrAttr(err)) + } + } g.Logger.Info("All done") } diff --git a/protocols/protocols.go b/protocols/protocols.go index e9a8b2a..d64be46 100644 --- a/protocols/protocols.go +++ b/protocols/protocols.go @@ -1,17 +1,18 @@ package protocols import ( - "bytes" "context" - "encoding/binary" "net" "strings" "github.com/mushorg/glutton/connection" "github.com/mushorg/glutton/producer" "github.com/mushorg/glutton/protocols/interfaces" + "github.com/mushorg/glutton/protocols/spicy" + spicyHandlers "github.com/mushorg/glutton/protocols/spicy/handlers" "github.com/mushorg/glutton/protocols/tcp" "github.com/mushorg/glutton/protocols/udp" + "github.com/spf13/viper" ) type TCPHandlerFunc func(ctx context.Context, conn net.Conn, md connection.Metadata) error @@ -72,6 +73,9 @@ func MapTCPProtocolHandlers(log interfaces.Logger, h interfaces.Honeypot) map[st protocolHandlers["mongodb"] = func(ctx context.Context, conn net.Conn, md connection.Metadata) error { return tcp.HandleMongoDB(ctx, conn, md, log, h) } + protocolHandlers["http"] = func(ctx context.Context, conn net.Conn, md connection.Metadata) error { + return tcp.HandleHTTP(ctx, conn, md, log, h) + } protocolHandlers["proxy_tcp"] = func(ctx context.Context, conn net.Conn, md connection.Metadata) error { return tcp.HandleProxyTCP(ctx, conn, md, log, h) } @@ -84,18 +88,17 @@ func MapTCPProtocolHandlers(log interfaces.Logger, h interfaces.Honeypot) map[st log.Debug("failed to peek connection", producer.ErrAttr(err)) return nil } - // poor mans check for HTTP request - httpMap := map[string]bool{"GET ": true, "POST": true, "HEAD": true, "OPTI": true, "CONN": true} - if _, ok := httpMap[strings.ToUpper(string(snip))]; ok { - return tcp.HandleHTTP(ctx, bufConn, md, log, h) - } - // poor mans check for RDP header - if bytes.Equal(snip, []byte{0x03, 0x00, 0x00, 0x2b}) { - return tcp.HandleRDP(ctx, bufConn, md, log, h) - } - // poor mans check for MongoDB header (checking msg length and validating opcodes) - messageLength := binary.LittleEndian.Uint32(snip) - if messageLength > 0 && messageLength <= 48*1024*1024 { + + // Uses a basic spicy parser to detect application protocol from tcp payload + if viper.GetBool("spicy.enabled") { + if protocol, ok := parseTCPProtocol(snip, log); ok { + switch protocol { + case "http": + return spicyHandlers.HandleHTTP(ctx, bufConn, md, log, h) + case "rdp": + return tcp.HandleRDP(ctx, bufConn, md, log, h) + } + } moreSample, bufConn, err := Peek(bufConn, 16) if err != nil { if err := conn.Close(); err != nil { @@ -104,22 +107,8 @@ func MapTCPProtocolHandlers(log interfaces.Logger, h interfaces.Honeypot) map[st log.Debug("failed to peek connection", producer.ErrAttr(err)) return nil } - if len(moreSample) == 16 { - opCode := binary.LittleEndian.Uint32(moreSample[12:16]) - validOpCodes := map[uint32]bool{ - 1: true, // OP_REPLY - 2001: true, // OP_UPDATE - 2002: true, // OP_INSERT - 2004: true, // OP_QUERY - 2005: true, // OP_GET_MORE - 2006: true, // OP_DELETE - 2007: true, // OP_KILL_CURSORS - 2012: true, // OP_COMPRESSED - 2013: true, // OP_MSG - } - if _, ok := validOpCodes[opCode]; ok { - return tcp.HandleMongoDB(ctx, bufConn, md, log, h) - } + if protocol, ok := parseTCPProtocol(moreSample, log); ok && protocol == "mongodb" { + return tcp.HandleMongoDB(ctx, bufConn, md, log, h) } } // fallback TCP handler @@ -127,3 +116,15 @@ func MapTCPProtocolHandlers(log interfaces.Logger, h interfaces.Honeypot) map[st } return protocolHandlers } + +func parseTCPProtocol(sample []byte, log interfaces.Logger) (string, bool) { + parsed, err := spicy.Parse("tcp", sample) + if err != nil { + log.Error("spicy tcp protocol parse error", producer.ErrAttr(err)) + return "", false + } + + protocol, ok := parsed.Fields["protocol"].(string) + protocol = strings.ToLower(strings.TrimSpace(protocol)) + return protocol, ok && protocol != "" +} diff --git a/protocols/protocols_test.go b/protocols/protocols_test.go index aedcd63..6a0fb2f 100644 --- a/protocols/protocols_test.go +++ b/protocols/protocols_test.go @@ -8,6 +8,7 @@ import ( "github.com/mushorg/glutton/connection" "github.com/mushorg/glutton/protocols/mocks" + "github.com/mushorg/glutton/protocols/spicy" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -56,3 +57,59 @@ func TestMapTCPProtocolHandlers(t *testing.T) { err := m["tcp"](ctx, conn, connection.Metadata{}) require.NoError(t, err, "expected no error from connection handler") } + +func TestParseTCPProtocol(t *testing.T) { + logger := &mocks.MockLogger{} + logger.EXPECT().Info(mock.Anything).Return().Maybe() + logger.EXPECT().Info(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return().Maybe() + logger.EXPECT().Error(mock.Anything, mock.Anything).Return().Maybe() + logger.EXPECT().Error(mock.Anything).Return().Maybe() + + if err := spicy.Initialize(logger); err != nil { + t.Skipf("Skipping test as Spicy initialization failed: %v", err) + } + + tests := []struct { + name string + sample []byte + protocol string + ok bool + }{ + { + name: "http", + sample: []byte("GET "), + protocol: "http", + ok: true, + }, + { + name: "rdp", + sample: []byte{0x03, 0x00, 0x00, 0x2b}, + protocol: "rdp", + ok: true, + }, + { + name: "mongodb", + sample: []byte{ + 0x10, 0x00, 0x00, 0x00, + 0x01, 0x02, 0x03, 0x04, + 0x00, 0x00, 0x00, 0x00, + 0xdd, 0x07, 0x00, 0x00, + }, + protocol: "mongodb", + ok: true, + }, + { + name: "invalid", + sample: []byte{0xff, 0x00, 0x01, 0x02}, + ok: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + protocol, ok := parseTCPProtocol(test.sample, logger) + require.Equal(t, test.ok, ok) + require.Equal(t, test.protocol, protocol) + }) + } +} diff --git a/protocols/spicy/Makefile b/protocols/spicy/Makefile new file mode 100644 index 0000000..d5de057 --- /dev/null +++ b/protocols/spicy/Makefile @@ -0,0 +1,32 @@ +GRAMMARS := $(wildcard parsers/*.spicy) + +.DEFAULT_GOAL := all + +GEN_CC := $(notdir $(GRAMMARS:.spicy=.cc)) +LINKER_CC := spicy_linker.cc +HEADERS := $(patsubst parsers/%.spicy,parsers/%.h,$(GRAMMARS)) + +SPICY_FLAGS := -g +CXX ?= clang++ +CXXFLAGS += -I/opt/spicy/include -std=c++20 -fPIC -O3 -DNDEBUG -fvisibility=hidden -I$(CURDIR)/parsers + +.SECONDARY: $(GEN_CC) $(LINKER_CC) $(HEADERS) + +%.cc: parsers/%.spicy + @echo "spicyc -c $< -> $@" + spicyc -c $(SPICY_FLAGS) $< -o $@ + +parsers/%.h: parsers/%.spicy + @echo "spicyc -P parsers/$* -o $@ $<" + spicyc -P parsers/$* -o $@ $< + +$(LINKER_CC): $(GRAMMARS) $(GEN_CC) + @echo "spicyc -l $(GRAMMARS) -> $@" + spicyc -l $(SPICY_FLAGS) $(GRAMMARS) -o $@ + +.PHONY: all +all: $(GEN_CC) $(LINKER_CC) $(HEADERS) + +.PHONY: clean +clean: + rm -f $(GEN_CC) $(LINKER_CC) $(HEADERS) \ No newline at end of file diff --git a/protocols/spicy/bridge.cpp b/protocols/spicy/bridge.cpp new file mode 100644 index 0000000..7bbc0bf --- /dev/null +++ b/protocols/spicy/bridge.cpp @@ -0,0 +1,394 @@ +#include "bridge.h" +#include +#include +#include +#include +#include +#include +#include +#include + +// Concrete parser modules are registered through generated Spicy linker code. +// Keep this bridge independent of parser-specific generated headers. +// tracks runtime init per thread +static thread_local bool t_thread_ready = false; + +// global mutex for runtime init sync +static std::mutex g_mutex; +static bool g_runtime_ready = false; + +// Note: Spicy/HILTI keep thread-local state, so we must call init() +// once on every OS thread that touches the runtime. The worker +// goroutine is pinned with runtime.LockOSThread() on the Go side +// initializes HILTI and Spicy for current thread +static inline void ensure_thread_ready() { + if (!t_thread_ready) { + hilti::rt::init(); // idempotent, but must run once per thread + spicy::rt::init(); // idempotent, but must run once per thread + t_thread_ready = true; + } +} + +// helper for early bail out if a previous allocation failed +// add_field_*() and dump_value() check this so that, after the first +// OOM, we stop allocating and just walk the data +static inline bool has_error(const ParsedData* d) { + return d && d->error_message; +} + +// safely duplicates C string, handling NULL input +// return NULL on OOM; callers must check +static char* strdup_safe(const char* s) { + if (!s) return nullptr; + size_t n = std::strlen(s) + 1; + char* p = static_cast(std::malloc(n)); + if (p) std::memcpy(p, s, n); + return p; +} + +// ensures space in the ParsedData fields array and returns +// reference to the next available field slot +// if realloc failed, sets error_message and returns a static dummy slot +// safe because of single worker thread model +static ParsedField& ensure_slot(ParsedData* dst) { + if (dst->field_count >= dst->capacity) { // need to grow the array + const int new_cap = dst->capacity ? dst->capacity * 2 : 16; + + // realloc safety check + void* mem = std::realloc(dst->fields, sizeof(ParsedField) * new_cap); + if (!mem) { // memory allocation failed + static ParsedField dummy; // placeholder + dst->error_message = strdup_safe("out of memory while growing ParsedField slot array"); + return dummy; + } + dst->fields = static_cast(mem); + dst->capacity = new_cap; + } + + ParsedField& f = dst->fields[dst->field_count++]; + f.name = nullptr; + f.value = nullptr; + f.is_binary = 0; + f.length = 0; + return f; +} + +// adds a string field to the ParsedData structure +static void add_field_str(ParsedData* dst, const std::string& name, const std::string& value){ + if (has_error(dst)) return; + + auto& f = ensure_slot(dst); + if (has_error(dst)) return; + + f.name = strdup_safe(name.c_str()); + if (!f.name) { + dst->error_message = strdup_safe("out of memory duplicating field name"); + --dst->field_count; // revert field count increment + return; + } + f.value = strdup_safe(value.c_str()); + if (!f.value) { + dst->error_message = strdup_safe("out of memory duplicating field value"); + std::free(f.name); + --dst->field_count; // revert field count increment + return; + } + f.is_binary = 0; + f.length = static_cast(value.size()); +} + +// adds a binary field to the ParsedData structure +static void add_field_bin(ParsedData* dst, const std::string& name, const uint8_t* data, size_t len) { + if (has_error(dst)) + return; + + if (len == 0) { + add_field_str(dst, name, ""); + return; + } + + auto& f = ensure_slot(dst); + if (has_error(dst)) + return; + + f.name = strdup_safe(name.c_str()); + if (!f.name) { + dst->error_message = strdup_safe("out of memory duplicating field name"); + --dst->field_count; + return; + } + + f.value = static_cast(std::malloc(len)); + if (!f.value) { + dst->error_message = strdup_safe("memory allocation failed for binary field"); + std::free(f.name); + --dst->field_count; + return; + } + std::memcpy(f.value, data, len); + f.is_binary = 1; + f.length = static_cast(len); +} + +// converts HILTI scalar value to its string representation and handles +// various HILTI type system values properly +static std::string scalar_to_string(const hilti::rt::type_info::Value& v) { + const auto& T = v.type(); + + switch (T.tag) { + case hilti::rt::TypeInfo::UnsignedInteger_uint64: + case hilti::rt::TypeInfo::SignedInteger_int64: + case hilti::rt::TypeInfo::UnsignedInteger_uint32: + case hilti::rt::TypeInfo::SignedInteger_int32: + case hilti::rt::TypeInfo::UnsignedInteger_uint16: + case hilti::rt::TypeInfo::SignedInteger_int16: + case hilti::rt::TypeInfo::UnsignedInteger_uint8: + case hilti::rt::TypeInfo::SignedInteger_int8: + case hilti::rt::TypeInfo::Real: + return std::to_string(v); + + case hilti::rt::TypeInfo::Bool: + return T.bool_->get(v) ? "true" : "false"; + + case hilti::rt::TypeInfo::String: + return T.string->get(v); + + case hilti::rt::TypeInfo::Enum: + return std::to_string(v); + + case hilti::rt::TypeInfo::Bytes: { + const auto& b = T.bytes->get(v); + return std::string(reinterpret_cast(b.data()), b.size()); + } + + case hilti::rt::TypeInfo::ValueReference: + return scalar_to_string(T.value_reference->value(v)); + + default: + return ""; + } +} + +static constexpr int kMaxDepth = 64; // maximum recursion depth for nested structures (prevents stack bombs) + +// Recursively flattens HILTI containers to "foo[3].bar" keys and stops at kMaxDepth +static void dump_value(ParsedData* dst, const std::string& prefix, const hilti::rt::type_info::Value& v, int depth = 0) { + if (depth > kMaxDepth) { + add_field_str(dst, prefix, ""); + return; + } + + const auto& T = v.type(); + + if (T.tag == hilti::rt::TypeInfo::ValueReference) { + dump_value(dst, prefix, T.value_reference->value(v)); + return; + } + + if (T.tag == hilti::rt::TypeInfo::Vector) { + size_t idx = 0; + for (const auto& elem : T.vector->iterate(v)) { + std::string key = prefix + "[" + std::to_string(idx++) + "]"; + dump_value(dst, key, elem, depth + 1); + } + return; + } + if (T.tag == hilti::rt::TypeInfo::Set) { + size_t idx = 0; + for (const auto& elem : T.set->iterate(v)) { + std::string key = prefix + "[" + std::to_string(idx++) + "]"; + dump_value(dst, key, elem, depth + 1); + } + return; + } + + if (T.tag == hilti::rt::TypeInfo::Map) { + auto* mt = hilti::rt::type_info::value::auxType(v); + + for (const auto& [k, val] : mt->iterate(v)) { + std::string kstr = scalar_to_string(k); + std::string key = prefix.empty() ? kstr : prefix + "." + kstr; + dump_value(dst, key, val, depth + 1); + } + return; + } + + if (T.tag == hilti::rt::TypeInfo::Optional) { + if (auto inner = T.optional->value(v)) + dump_value(dst, prefix, inner, depth + 1); + else + add_field_str(dst, prefix, ""); + return; + } + + if (T.tag == hilti::rt::TypeInfo::Struct) { + for (const auto& [info, field] : T.struct_->iterate(v)) { + std::string key = prefix.empty() ? info.name : prefix + "." + info.name; + dump_value(dst, key, field, depth + 1); + } + return; + } + + if (T.tag == hilti::rt::TypeInfo::Bytes) { + const auto& b = T.bytes->get(v); + + bool printable = true; + for (auto c : b) { + if (c < 0x20 || c > 0x7e) { printable = false; break; } + } + + if (printable && b.size() <= 256) { + add_field_str(dst, prefix, std::string(reinterpret_cast(b.data()), b.size())); + } + else { + add_field_bin(dst, prefix, reinterpret_cast(b.data()), b.size()); + } + return; + } + + add_field_str(dst, prefix, scalar_to_string(v)); +} + +// initializes HILTI and Spicy runtimes globally +void spicy_init() { + std::lock_guard lock(g_mutex); + if (g_runtime_ready) + return; + + hilti::rt::init(); + spicy::rt::init(); + g_runtime_ready = true; +} + +// cleans up HILTI and Spicy runtimes globally +void spicy_cleanup() { + std::lock_guard lock(g_mutex); + if (!g_runtime_ready) + return; + + spicy::rt::done(); + hilti::rt::done(); + g_runtime_ready = false; +} + +// checks if the Spicy runtime is initialized +int spicy_is_initialized() { + std::lock_guard lock(g_mutex); + return g_runtime_ready ? 1 : 0; +} + +// lists all available Spicy parsers and returns their names +char** spicy_list_parsers(int* count) { + std::lock_guard lock(g_mutex); + + ensure_thread_ready(); + + if (!g_runtime_ready) { + if (count) *count = 0; + return nullptr; + } + + if (!count) return nullptr; + + try { + spicy::rt::Driver drv; + std::stringstream ss; + drv.listParsers(ss); + + std::vector names; + std::string line; + while (std::getline(ss, line)) + if (line.find("::") != std::string::npos) names.emplace_back(line); + + *count = static_cast(names.size()); + if (*count == 0) + return nullptr; + + char** out = static_cast(std::malloc(sizeof(char*) * *count)); + for (int i = 0; i < *count; ++i) + out[i] = strdup_safe(names[i].c_str()); + + return out; + } + catch (const std::exception& e) { + *count = 0; + return nullptr; + } +} + +// parses data using a specified Spicy parser and returns the parsed data +// called only from the single locked OS thread in the Go worker +ParsedData* spicy_parse_generic(const char* parser_name, const unsigned char* data, int length) { + if (!parser_name || !data || length <= 0) + return nullptr; + + auto* res = static_cast(std::calloc(1, sizeof(ParsedData))); + if (!res) + return nullptr; + + res->protocol_name = strdup_safe(parser_name); + + std::lock_guard lock(g_mutex); + + ensure_thread_ready(); + + if (!g_runtime_ready) { + res->error_message = strdup_safe("runtime not initialized"); + return res; + } + + try { + spicy::rt::Driver drv; + auto parser = drv.lookupParser(parser_name); + + if (!parser) { + res->error_message = strdup_safe("parser not found"); + return res; + } + + std::stringstream in; + in.write(reinterpret_cast(data), length); + + auto unit = drv.processInput(**parser, in); + + if (unit && unit->value()) { + dump_value(res, "", unit->value(), 0); + } + else { + res->error_message = strdup_safe("no value returned"); + } + } + catch (const std::exception& e) { + res->error_message = strdup_safe(e.what()); + } + catch (...) { + res->error_message = strdup_safe("unknown C++ exception"); + } + + return res; +} + +// frees the memory allocated for ParsedData and its fields +void spicy_free_parsed_data(ParsedData* d) { + if (!d) + return; + + for (int i = 0; i < d->field_count; ++i) { + std::free(d->fields[i].name); + std::free(d->fields[i].value); + } + std::free(d->fields); + std::free(d->protocol_name); + std::free(d->error_message); + std::free(d); +} + +// frees the memory allocated for a list of parser names +void spicy_free_parser_list(char** p, int n) { + if (!p) + return; + + for (int i = 0; i < n; ++i) + std::free(p[i]); + std::free(p); +} diff --git a/protocols/spicy/bridge.h b/protocols/spicy/bridge.h new file mode 100644 index 0000000..e2d6b1f --- /dev/null +++ b/protocols/spicy/bridge.h @@ -0,0 +1,63 @@ +#ifndef GLUTTON_SPICY_BRIDGE_H +#define GLUTTON_SPICY_BRIDGE_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + char* name; // name of field + char* value; // value of field. + int is_binary; // flag (0 or 1) indicating if value is raw bytes + int length; // length of value, essential for binary data +} ParsedField; + +typedef struct { + ParsedField* fields; + int field_count; + int capacity; + char* protocol_name; + char* error_message; +} ParsedData; + +/** + * @brief Initializes the Spicy and HILTI runtimes. + */ +void spicy_init(); + +/** + * @brief Checks if the Spicy runtime is initialized. + * returns 1 if initialized, 0 if not. + */ +int spicy_is_initialized(); + +/** + * @brief Cleans up and shuts down the Spicy and HILTI runtimes. + */ +void spicy_cleanup(); + +/** + * @brief Lists all available (compiled-in) Spicy parsers. + */ +char** spicy_list_parsers(int* count); + +/** + * @brief The main generic parsing function. + */ +ParsedData* spicy_parse_generic(const char* parser_name, const unsigned char* data, int length); + +/** + * @brief Frees the memory allocated for a ParsedData struct. + */ +void spicy_free_parsed_data(ParsedData* data); + +/** + * @brief Frees the memory allocated for the parser list returned by spicy_list_parsers. + */ +void spicy_free_parser_list(char** parsers, int count); + +#ifdef __cplusplus +} +#endif + +#endif // GLUTTON_SPICY_BRIDGE_H \ No newline at end of file diff --git a/protocols/spicy/handlers/http.go b/protocols/spicy/handlers/http.go new file mode 100644 index 0000000..3e6f759 --- /dev/null +++ b/protocols/spicy/handlers/http.go @@ -0,0 +1,207 @@ +package handlers + +import ( + "bytes" + "context" + "encoding/hex" + "encoding/json" + "fmt" + "log/slog" + "net" + "strconv" + "strings" + + "github.com/mushorg/glutton/connection" + "github.com/mushorg/glutton/producer" + "github.com/mushorg/glutton/protocols/interfaces" + "github.com/mushorg/glutton/protocols/spicy" + "github.com/mushorg/glutton/protocols/tcp" +) + +// Identical implementation of the original Go HTTP handler, but using Spicy for parsing +// I've tried to keep the logs and responses as close to the original as possible + +func sendJSON(conn net.Conn, b []byte) error { + _, err := conn.Write( + append([]byte(fmt.Sprintf("HTTP/1.1 200 OK\r\nContent-Length:%d\r\n\r\n", len(b))), b...), + ) + return err +} + +func writePlainOK(conn net.Conn) error { + _, err := conn.Write([]byte("HTTP/1.1 200 OK\r\n\r\n")) + return err +} + +func handleEthereumRPC(body []byte, conn net.Conn) bool { + if !bytes.Contains(body, []byte("eth_blockNumber")) { + return false + } + resp := struct { + JSONRPC string `json:"jsonrpc"` + ID int `json:"id"` + Result string `json:"result"` + }{ + JSONRPC: "2.0", + ID: 0, + Result: "0x2ecd9e", + } + b, _ := json.Marshal(resp) + _ = sendJSON(conn, b) + return true +} + +func handleYarnNewApplication(method, uri string, conn net.Conn) bool { + if method != "POST" || !strings.Contains(uri, "cluster/apps/new-application") { + return false + } + resp, _ := json.Marshal(&struct { + ApplicationID string `json:"application-id"` + MaximumResourceCapability interface{} `json:"maximum-resource-capability"` + }{ + ApplicationID: "application_1527144634877_20465", + MaximumResourceCapability: struct { + Memory int `json:"memory"` + VCores int `json:"vCores"` + }{Memory: 16384, VCores: 8}, + }) + _ = sendJSON(conn, resp) + return true +} + +func handleWallet(uri string, conn net.Conn) bool { + if !strings.Contains(uri, "wallet") { + return false + } + body := []byte(`[[""]]`) + header := fmt.Sprintf("HTTP/1.1 200 OK\r\nContent-Length:%d\r\n\r\n", len(body)) + conn.Write([]byte(header)) + conn.Write(body) + return true +} + +func handleDockerAPIVersion(path string, conn net.Conn, log interfaces.Logger) bool { + if !strings.HasPrefix(path, "/v1.16/version") { + return false + } + data, err := tcp.Res.ReadFile("resources/docker_api.json") + if err != nil { + log.Error("failed to read docker_api.json", producer.ErrAttr(err)) + return false + } else { + conn.Write(append([]byte(fmt.Sprintf("HTTP/1.1 200 OK\r\nContent-Length:%d\r\n\r\n", len(data))), data...)) + } + return true +} + +func handleCitrixSMB(path string, conn net.Conn) bool { + if !strings.HasPrefix(path, "/vpn/") { + return false + } + headers := `Server: Apache +X-Frame-Options: SAMEORIGIN +Last-Modified: Thu, 28 Nov 2019 20:19:22 GMT +ETag: "53-5986dd42b0680" +Accept-Ranges: bytes +Content-Length: 93 +X-XSS-Protection: 1; mode=block +X-Content-Type-Options: nosniff +Content-Type: text/plain; charset=UTF-8` + smbCfg := "\r\n\r\n[global]\r\n\tencrypt passwords = yes\r\n\tname resolve order = lmhosts wins host bcast\r\n" + conn.Write([]byte("HTTP/1.1 200 OK\r\n" + headers + smbCfg)) + return true +} + +func handleVMwareSend(ctx context.Context, body []byte, uri string, md connection.Metadata, log interfaces.Logger, hp interfaces.Honeypot) bool { + if !strings.Contains(uri, "hyper/send") || len(body) == 0 { + return false + } + parts := strings.Split(string(body), " ") + if len(parts) < 11 { + return false + } + c, err := net.Dial("tcp", parts[9]+":"+parts[10]) + if err != nil { + log.Error("vmware-send dial failed", producer.ErrAttr(err)) + return true + } + go func() { + if err := tcp.HandleTCP(ctx, c, md, log, hp); err != nil { + log.Error("vmware-send TCP relay error", producer.ErrAttr(err)) + } + }() + return true +} + +func HandleHTTP(ctx context.Context, conn net.Conn, md connection.Metadata, log interfaces.Logger, hp interfaces.Honeypot) error { + defer conn.Close() + + payload, err := spicy.ReadInitialBytes("http", conn) + if err != nil { + return err + } + if len(payload) == 0 { + return nil + } + + parsed, err := spicy.Parse("http", payload) // parse the HTTP request using Spicy + if err != nil { + log.Error("spicy parse error", producer.ErrAttr(err)) + _ = hp.ProduceTCP("spicy-http-failed", conn, md, payload, + map[string]string{"error": err.Error()}) + return err + } + + method, _ := parsed.Fields["method"].(string) + method = strings.ToUpper(method) + uriRaw, _ := parsed.Fields["uri.raw"].(string) + path, _ := parsed.Fields["uri.path"].(string) + if path == "" { + path = uriRaw + } + query, _ := parsed.Fields["uri.query"].(string) + version, _ := parsed.Fields["version.number"].(string) + + var body []byte + if v, ok := parsed.Fields["body.content"]; ok { + switch b := v.(type) { + case []byte: + body = b + case string: + body = []byte(b) + } + } + + srcHost, srcPort, _ := net.SplitHostPort(conn.RemoteAddr().String()) + log.Info(fmt.Sprintf("HTTP %s %s request handled: %s", version, method, path), // added "version" as a proof of concept, not identical to the original pure Go parser + slog.String("handler", "spicy-http"), + slog.String("dest_port", strconv.Itoa(int(md.TargetPort))), + slog.String("src_ip", srcHost), + slog.String("src_port", srcPort), + slog.String("path", path), + slog.String("query", query), + ) + + if len(body) > 0 { + max := len(body) + if max > 1024 { + max = 1024 + } + log.Info("HTTP payload:\n" + hex.Dump(body[:max])) + } + + _ = hp.ProduceTCP("http", conn, md, payload, parsed) + + handled := false + switch method { + case "POST": + handled = handleEthereumRPC(body, conn) || handleYarnNewApplication(method, uriRaw, conn) + } + + handled = handled || handleWallet(uriRaw, conn) || handleDockerAPIVersion(path, conn, log) || handleCitrixSMB(path, conn) || handleVMwareSend(ctx, body, uriRaw, md, log, hp) + + if !handled { + _ = writePlainOK(conn) + } + return nil +} diff --git a/protocols/spicy/handlers/http_test.go b/protocols/spicy/handlers/http_test.go new file mode 100644 index 0000000..921555a --- /dev/null +++ b/protocols/spicy/handlers/http_test.go @@ -0,0 +1,199 @@ +package handlers + +import ( + "bytes" + "context" + "fmt" + "net" + "sync" + "testing" + + "github.com/mushorg/glutton/connection" + "github.com/mushorg/glutton/protocols/mocks" + "github.com/mushorg/glutton/protocols/spicy" + "github.com/mushorg/glutton/rules" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +type mockConn struct { + net.Conn + readBuf *bytes.Buffer + writeBuf *bytes.Buffer + remoteAddr net.Addr + closed bool +} + +func newMockConn(data string) *mockConn { + return &mockConn{ + readBuf: bytes.NewBufferString(data), + writeBuf: &bytes.Buffer{}, + remoteAddr: &net.TCPAddr{IP: net.ParseIP("192.168.1.100"), Port: 12345}, + } +} + +func (m *mockConn) Read(b []byte) (n int, err error) { return m.readBuf.Read(b) } +func (m *mockConn) Write(b []byte) (n int, err error) { return m.writeBuf.Write(b) } +func (m *mockConn) Close() error { m.closed = true; return nil } +func (m *mockConn) RemoteAddr() net.Addr { return m.remoteAddr } +func (m *mockConn) Written() string { return m.writeBuf.String() } + +func createMockLogger() *mocks.MockLogger { + logger := &mocks.MockLogger{} + + // generic expectations that handle all variations + logger.EXPECT().Info(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return().Maybe() + logger.EXPECT().Info(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return().Maybe() + logger.EXPECT().Info(mock.Anything, mock.Anything).Return().Maybe() + logger.EXPECT().Info(mock.Anything).Return().Maybe() + logger.EXPECT().Error(mock.Anything, mock.Anything).Return().Maybe() + logger.EXPECT().Error(mock.Anything).Return().Maybe() + + return logger +} + +// initialize Spicy once for all tests +var spicyInitOnce sync.Once + +func ensureSpicyInitialized() { + spicyInitOnce.Do(func() { + logger := createMockLogger() + spicy.Initialize(logger) + }) +} + +func buildHTTPRequest(method, target, body string, headers ...string) string { + request := fmt.Sprintf("%s %s HTTP/1.1\r\nHost: example.com\r\n", method, target) + for _, header := range headers { + request += header + "\r\n" + } + if body != "" { + request += fmt.Sprintf("Content-Length: %d\r\n", len(body)) + } + return request + "\r\n" + body +} + +func runHTTPHandler(t *testing.T, request string) *mockConn { + t.Helper() + ensureSpicyInitialized() + + conn := newMockConn(request) + logger := createMockLogger() + honeypot := &mocks.MockHoneypot{} + honeypot.EXPECT().ProduceTCP("http", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + + md := connection.Metadata{ + TargetPort: 80, + Rule: &rules.Rule{Target: "http"}, + } + + err := HandleHTTP(context.Background(), conn, md, logger, honeypot) + require.NoError(t, err) + require.True(t, conn.closed) + + logger.AssertExpectations(t) + honeypot.AssertExpectations(t) + + return conn +} + +func TestHandleHTTPBasicGET(t *testing.T) { + conn := runHTTPHandler(t, buildHTTPRequest("GET", "/test", "")) + + require.Contains(t, conn.Written(), "HTTP/1.1 200 OK") +} + +func TestHandleHTTPResponseBranches(t *testing.T) { + ethereumBody := `{"jsonrpc":"2.0","method":"eth_blockNumber","id":1}` + + tests := []struct { + name string + request string + contains string + }{ + { + name: "Default", + request: buildHTTPRequest("GET", "/test", ""), + contains: "HTTP/1.1 200 OK\r\n\r\n", + }, + { + name: "Wallet", + request: buildHTTPRequest("GET", "/wallet", ""), + contains: `[[""]]`, + }, + { + name: "Docker", + request: buildHTTPRequest("GET", "/v1.16/version", ""), + contains: `"ApiVersion":"1.41"`, + }, + { + name: "YARN", + request: buildHTTPRequest("POST", "/ws/v1/cluster/apps/new-application", `{}`), + contains: `"application-id":"application_1527144634877_20465"`, + }, + { + name: "Ethereum", + request: buildHTTPRequest("POST", "/", ethereumBody), + contains: `"result":"0x2ecd9e"`, + }, + { + name: "Citrix", + request: buildHTTPRequest("GET", "/vpn/index.html", ""), + contains: "[global]", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + require.Contains(t, runHTTPHandler(t, test.request).Written(), test.contains) + }) + } +} + +func TestHandleHTTPUsesParsedQuery(t *testing.T) { + conn := runHTTPHandler(t, buildHTTPRequest("GET", "/test/path?x=1&y=two", "")) + + require.Contains(t, conn.Written(), "HTTP/1.1 200 OK") +} + +func TestHandleHTTPWithBody(t *testing.T) { + conn := runHTTPHandler(t, buildHTTPRequest("POST", "/api", `{"test":true}`)) + + require.True(t, conn.closed) +} + +func TestHandleHTTPMalformedRequest(t *testing.T) { + ensureSpicyInitialized() + + malformedRequest := "GET /path\r\nHost: test\r\n\r\n" + conn := newMockConn(malformedRequest) + + logger := createMockLogger() + honeypot := &mocks.MockHoneypot{} + honeypot.EXPECT().ProduceTCP("spicy-http-failed", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + + md := connection.Metadata{TargetPort: 80} + ctx := context.Background() + + err := HandleHTTP(ctx, conn, md, logger, honeypot) + require.Error(t, err) + require.True(t, conn.closed) + + logger.AssertExpectations(t) + honeypot.AssertExpectations(t) +} + +func TestHandleHTTPEmptyRequest(t *testing.T) { + ensureSpicyInitialized() + + conn := newMockConn("") + logger := createMockLogger() + honeypot := &mocks.MockHoneypot{} + + md := connection.Metadata{TargetPort: 80} + ctx := context.Background() + + err := HandleHTTP(ctx, conn, md, logger, honeypot) + require.Error(t, err) // empty request should error (EOF when trying to read) + require.True(t, conn.closed) +} diff --git a/protocols/spicy/helpers.go b/protocols/spicy/helpers.go new file mode 100644 index 0000000..30805f9 --- /dev/null +++ b/protocols/spicy/helpers.go @@ -0,0 +1,159 @@ +package spicy + +import ( + "bufio" + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "net" + "regexp" + "strconv" + "strings" +) + +// package level regex to match conlen header in HTTP requests +var contentLenRE = regexp.MustCompile(`(?i)Content-Length:\s*(\d+)`) + +// reads protocol-specific initial data from a network connection and +// returns the complete protocol message as a byte slice. +func ReadInitialBytes(protocol string, conn net.Conn) ([]byte, error) { + switch protocol { + + case "http": + const maxHTTPBody = 1 << 20 // 1 MiB limit (abuse cap, not HTTP limit) + + r := bufio.NewReader(conn) + raw := make([]byte, 0, 4096) + + for { + line, err := r.ReadBytes('\n') + if err != nil { + return nil, err + } + raw = append(raw, line...) + if bytes.Equal(line, []byte("\r\n")) { + break + } + } + + var clen int + if m := contentLenRE.FindSubmatch(raw); m != nil { + clen, _ = strconv.Atoi(string(m[1])) + } + + if clen > maxHTTPBody { + return nil, fmt.Errorf("Content-Length %d exceeds maximum %d", clen, maxHTTPBody) + } + + if clen > 0 { + body := make([]byte, clen) + if _, err := io.ReadFull(r, body); err != nil { + return nil, err + } + raw = append(raw, body...) + } + return raw, nil + + case "dns": + var lenBuf [2]byte + if _, err := io.ReadFull(conn, lenBuf[:]); err != nil { + return nil, err + } + l := int(binary.BigEndian.Uint16(lenBuf[:])) + if l == 0 || l > 64*1024 { + return nil, errors.New("suspicious DNS length") + } + p := make([]byte, l) + _, err := io.ReadFull(conn, p) + return p, err + + default: + buf := make([]byte, 8192) + n, err := conn.Read(buf) + return buf[:n], err + } +} + +// converts a flat map with dot notation keys into a nested map structure. +// created initially as a Spicy helper to handle nested data structures +// currently unused, kept for potential future use +func NestedFromFlat(flat map[string]interface{}) map[string]interface{} { + root := map[string]interface{}{} + + for k, v := range flat { + cur := root + parts := strings.Split(k, ".") + + for i, p := range parts { + if b := strings.Index(p, "["); b != -1 { + base := p[:b] + e := strings.Index(p[b:], "]") + idx, _ := strconv.Atoi(p[b+1 : b+e]) + + slice, ok := cur[base].([]interface{}) + if !ok { + slice = make([]interface{}, idx+1) + cur[base] = slice + } else if idx >= len(slice) { + slice = append(slice, make([]interface{}, idx+1-len(slice))...) + cur[base] = slice + } + + if i == len(parts)-1 { + slice[idx] = v + } else { + if slice[idx] == nil { + slice[idx] = map[string]interface{}{} + } + cur, ok = slice[idx].(map[string]interface{}) + if !ok { + return nil + } + } + } else { + if i == len(parts)-1 { + cur[p] = v + } else { + if _, ok := cur[p]; !ok { + cur[p] = map[string]interface{}{} + } + m, ok := cur[p].(map[string]interface{}) + if !ok { + return nil + } + cur = m + } + } + } + } + return root +} + +// retrieves a string value from a nested map using a path with dot notation +// currently unused, kept for potential future use +func GetDeepStr(m map[string]interface{}, path ...string) string { + for _, p := range path { + parts := strings.Split(p, ".") + cur := m + for i, seg := range parts { + v, ok := cur[seg] + if !ok { + break + } + if i == len(parts)-1 { + if s, ok := v.(string); ok { + return s + } + } else { + if nxt, ok := v.(map[string]interface{}); ok { + cur = nxt + } else { + break + } + } + } + } + return "" +} diff --git a/protocols/spicy/parser.go b/protocols/spicy/parser.go new file mode 100644 index 0000000..11d9437 --- /dev/null +++ b/protocols/spicy/parser.go @@ -0,0 +1,229 @@ +package spicy + +/* +#cgo CXXFLAGS: -I/opt/spicy/include -I${SRCDIR}/parsers -std=c++20 -fPIC -O3 -DNDEBUG -fvisibility=hidden +#cgo LDFLAGS: -L/opt/spicy/lib -lspicy-rt -lhilti-rt -lz -lpthread -ldl "-Wl,-rpath,/opt/spicy/lib" +#include +#include "bridge.h" +*/ +import "C" + +import ( + "errors" + "fmt" + "runtime" + "strings" + "sync" + "time" + "unsafe" + + "github.com/mushorg/glutton/protocols/interfaces" +) + +type workerCmdKind int8 + +const ( + cmdInitAndList workerCmdKind = iota + 1 + cmdParse + cmdShutdown +) + +type workerCmd struct { + kind workerCmdKind + parser string + data []byte + replyChan chan any +} + +var ( + workerOnce sync.Once + cmdCh chan workerCmd +) + +// initializes the Spicy worker thread if not already started +func startWorker() { + workerOnce.Do(func() { + cmdCh = make(chan workerCmd) + go func() { + runtime.LockOSThread() // lock to OS thread for C++ runtime thread-local storage (mandatory for Spicy/HILTI TLS) + defer runtime.UnlockOSThread() + + for cmd := range cmdCh { + switch cmd.kind { + case cmdInitAndList: + C.spicy_init() + + cnt := C.int(0) + pp := C.spicy_list_parsers(&cnt) + list := []string{} + if pp != nil && cnt > 0 { + ptrs := (*[1 << 30]*C.char)(unsafe.Pointer(pp))[:cnt:cnt] + for _, p := range ptrs { + list = append(list, C.GoString(p)) + } + C.spicy_free_parser_list(pp, cnt) + } + cmd.replyChan <- list + + case cmdParse: + cn := C.CString(cmd.parser) + cres := C.spicy_parse_generic( + cn, + (*C.uchar)(unsafe.Pointer(&cmd.data[0])), + C.int(len(cmd.data)), + ) + C.free(unsafe.Pointer(cn)) + cmd.replyChan <- cres + + case cmdShutdown: + var err error + func() { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic during cleanup: %v", r) + } + }() + C.spicy_cleanup() + }() + cmd.replyChan <- err + close(cmd.replyChan) + return + } + } + }() + }) +} + +var ( + initOnce sync.Once // ensures Spicy runtime is initialized only once + registeredParsers = make(map[string]string) + parsersMutex sync.RWMutex // protects access to registeredParsers +) + +var initErr error + +func Initialize(logger interfaces.Logger) error { + initOnce.Do(func() { + // starts Spicy worker thread + startWorker() + + resp := make(chan any, 1) + cmdCh <- workerCmd{kind: cmdInitAndList, replyChan: resp} + names := (<-resp).([]string) + + if C.spicy_is_initialized() == 0 { + initErr = errors.New("failed to initialise Spicy runtime") + logger.Error(initErr.Error()) + return + } + logger.Info("Spicy runtime initialised successfully") + + parsersMutex.Lock() + for _, raw := range names { + raw = strings.TrimSpace(raw) + + if raw == "" || !strings.Contains(raw, "::") { + continue + } + + // protocol names look like "HTTP::Request", so we split on "::" + parts := strings.SplitN(raw, "::", 2) + proto := strings.ToLower(strings.TrimSpace(parts[0])) + canonical := strings.TrimSpace(raw) + + if _, ok := registeredParsers[proto]; !ok { + registeredParsers[proto] = canonical + logger.Info("registered Spicy parser", "protocol", proto, "parser", canonical) + } + } + parsersMutex.Unlock() + }) + return initErr +} + +// represents the result of parsing protocol data with Spicy +type ParsedData struct { + Protocol string `json:"protocol"` + Fields map[string]interface{} `json:"fields"` + Error error `json:"-"` +} + +const parseTimeout = 10 * time.Second + +// analyzes protocol data using the appropriate Spicy parser +// the parser is automatically selected based on the protocol name +func Parse(proto string, data []byte) (*ParsedData, error) { + parsersMutex.RLock() + key := strings.ToLower(strings.TrimSpace(proto)) + name, ok := registeredParsers[key] // parser lookup + parsersMutex.RUnlock() + if !ok { + return nil, fmt.Errorf("no Spicy parser registered for %q", proto) + } + + if len(data) == 0 { + return nil, errors.New("input data is empty") + } + + resp := make(chan any, 1) + cmdCh <- workerCmd{kind: cmdParse, parser: name, data: data, replyChan: resp} + + var raw any + select { + case raw = <-resp: // normal path + case <-time.After(parseTimeout): // worker stalled + // drain resp to free C memory even after we have returned + go func() { + if r := <-resp; r != nil { + if p, ok := r.(*C.ParsedData); ok { + C.spicy_free_parsed_data(p) + } + } + }() + return nil, fmt.Errorf("Spicy parse timed-out after %s", parseTimeout) + } + + if raw == nil { + return nil, errors.New("Spicy parse failed: no response received") + } + + cRes, ok := raw.(*C.ParsedData) + if !ok { + return nil, errors.New("internal type assertion failed") + } + defer C.spicy_free_parsed_data(cRes) + + out := &ParsedData{Protocol: proto, Fields: map[string]interface{}{}} + if cRes.error_message != nil { + err := errors.New(C.GoString(cRes.error_message)) + out.Error = err + return out, err + } + + // extract parsed fields + if cRes.fields != nil && cRes.field_count > 0 { + fs := (*[1 << 30]C.ParsedField)(unsafe.Pointer(cRes.fields))[:cRes.field_count:cRes.field_count] + for _, f := range fs { + k := C.GoString(f.name) + if f.is_binary != 0 { + // binary + out.Fields[k] = C.GoBytes(unsafe.Pointer(f.value), f.length) + } else { + // string + out.Fields[k] = C.GoString(f.value) + } + } + } + return out, nil +} + +// shuts down the Spicy runtime and releases all associated resources +// it's probably safe to call multiple times +func Cleanup() error { + resp := make(chan any, 1) + cmdCh <- workerCmd{kind: cmdShutdown, replyChan: resp} + if v := <-resp; v != nil { + return v.(error) + } + return nil +} diff --git a/protocols/spicy/parser_test.go b/protocols/spicy/parser_test.go new file mode 100644 index 0000000..291c710 --- /dev/null +++ b/protocols/spicy/parser_test.go @@ -0,0 +1,161 @@ +package spicy + +import ( + "sync" + "testing" + + "github.com/mushorg/glutton/protocols/mocks" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func createMockLogger() *mocks.MockLogger { + logger := &mocks.MockLogger{} + logger.EXPECT().Info(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return().Maybe() + logger.EXPECT().Info(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return().Maybe() + logger.EXPECT().Info(mock.Anything, mock.Anything).Return().Maybe() + logger.EXPECT().Info(mock.Anything).Return().Maybe() + logger.EXPECT().Error(mock.Anything, mock.Anything).Return().Maybe() + logger.EXPECT().Error(mock.Anything).Return().Maybe() + return logger +} + +var spicyInitOnce sync.Once +var spicyInitErr error + +func ensureSpicyInitialized() { + spicyInitOnce.Do(func() { + logger := createMockLogger() + spicyInitErr = Initialize(logger) + }) +} + +func TestInitialize(t *testing.T) { + ensureSpicyInitialized() + + logger := createMockLogger() + + err1 := Initialize(logger) + err2 := Initialize(logger) + + if spicyInitErr != nil { + require.Error(t, err1) + require.Error(t, err2) + t.Logf("Initialize failed (expected if Spicy not available): %v", err1) + } else { + require.NoError(t, err1) + require.NoError(t, err2) + } + + logger.AssertExpectations(t) +} + +func TestParseHTTPRequest(t *testing.T) { + ensureSpicyInitialized() + + if spicyInitErr != nil { + t.Skipf("Skipping test as Spicy initialization failed: %v", spicyInitErr) + } + + httpRequest := "POST /test/path?x=1&y=two HTTP/1.1\r\nHost: example.com:8080\r\nUser-Agent: glutton-test\r\nContent-Type: application/json\r\nContent-Length: 4\r\n\r\ntest" + result, err := Parse("http", []byte(httpRequest)) + + if err != nil { + if err.Error() == "no Spicy parser registered for \"http\"" { + t.Skip("HTTP parser not available in this build") + return + } + require.NoError(t, err, "Unexpected parsing error") + } + + require.NotNil(t, result) + require.Equal(t, "http", result.Protocol) + require.NotNil(t, result.Fields) + require.NoError(t, result.Error) + + require.Equal(t, "POST", result.Fields["method"]) + require.Equal(t, "/test/path?x=1&y=two", result.Fields["uri.raw"]) + require.Equal(t, "/test/path", result.Fields["uri.path"]) + require.Equal(t, "x=1&y=two", result.Fields["uri.query"]) + require.Equal(t, "1.1", result.Fields["version.number"]) + require.Equal(t, "Host", result.Fields["headers[0].name"]) + require.Equal(t, "example.com:8080", result.Fields["headers[0].value"]) + require.Equal(t, "test", result.Fields["body.content"]) + require.NotContains(t, result.Fields, "header.host") + require.NotContains(t, result.Fields, "header.user_agent") + require.NotContains(t, result.Fields, "header.content_type") + require.NotContains(t, result.Fields, "header.content_length") + require.NotContains(t, result.Fields, "host") + require.NotContains(t, result.Fields, "port_") +} + +func TestParseHTTPRequestAbsoluteTarget(t *testing.T) { + ensureSpicyInitialized() + + if spicyInitErr != nil { + t.Skipf("Skipping test as Spicy initialization failed: %v", spicyInitErr) + } + + httpRequest := "GET http://example.com/test/path?x=1 HTTP/1.1\r\nHost: example.com\r\n\r\n" + result, err := Parse("http", []byte(httpRequest)) + + if err != nil { + if err.Error() == "no Spicy parser registered for \"http\"" { + t.Skip("HTTP parser not available in this build") + return + } + require.NoError(t, err, "Unexpected parsing error") + } + + require.NotNil(t, result) + require.Equal(t, "http://example.com/test/path?x=1", result.Fields["uri.raw"]) + require.Equal(t, "http://example.com/test/path", result.Fields["uri.path"]) + require.Equal(t, "x=1", result.Fields["uri.query"]) +} + +func TestParseUnknownProtocol(t *testing.T) { + ensureSpicyInitialized() + + if spicyInitErr != nil { + t.Skipf("Skipping test as Spicy initialization failed: %v", spicyInitErr) + } + + result, err := Parse("unknown-protocol", []byte("test data")) + + require.Error(t, err) + require.Nil(t, result) + require.Contains(t, err.Error(), "no Spicy parser registered") +} + +func TestParseEmptyData(t *testing.T) { + ensureSpicyInitialized() + + if spicyInitErr != nil { + t.Skipf("Skipping test as Spicy initialization failed: %v", spicyInitErr) + } + + result, err := Parse("http", []byte{}) + + require.Error(t, err) + require.Nil(t, result) + require.Contains(t, err.Error(), "input data is empty") +} + +func TestParseNilData(t *testing.T) { + ensureSpicyInitialized() + + if spicyInitErr != nil { + t.Skipf("Skipping test as Spicy initialization failed: %v", spicyInitErr) + } + + result, err := Parse("http", nil) + + require.Error(t, err) + require.Nil(t, result) + require.Contains(t, err.Error(), "input data is empty") +} + +func TestCleanup(t *testing.T) { + err := Cleanup() + require.NoError(t, err) +} diff --git a/protocols/spicy/parsers/http.spicy b/protocols/spicy/parsers/http.spicy new file mode 100644 index 0000000..0566380 --- /dev/null +++ b/protocols/spicy/parsers/http.spicy @@ -0,0 +1,49 @@ +module HTTP; + +const Token = /[^ \t\r\n]+/; +const HeaderName = /[^:\r\n]+/; +const WhiteSpace = /[ \t]+/; +const OptionalWhiteSpace = /[ \t]*/; +const NewLine = /\r?\n/; +const RestOfLine = /[^\r\n]*/; + +type Version = unit { + : /HTTP\//; + number: /[0-9]+\.[0-9]*/; +}; + +type URI = unit { + # parse path/query fields for handlers. + raw: Token { + local parts = $$.split1(b"?"); + self.path = parts[0]; + self.query = parts[1]; + } + + var path: bytes; + var query: bytes; +}; + +type Header = unit { + name: HeaderName; + : /:/; + : OptionalWhiteSpace; + value: RestOfLine; + : NewLine; +}; + +type Body = unit { + content: bytes &eod; +}; + +public type Request = unit { + method: Token; + : WhiteSpace; + uri: URI; + : WhiteSpace; + version: Version; + : NewLine; + headers: Header[]; + : NewLine; + body: Body; +}; diff --git a/protocols/spicy/parsers/tcp.spicy b/protocols/spicy/parsers/tcp.spicy new file mode 100644 index 0000000..3b1a53d --- /dev/null +++ b/protocols/spicy/parsers/tcp.spicy @@ -0,0 +1,35 @@ +module TCP; + +import spicy; + +const MaxMongoDBMessageLength: uint64 = 48 * 1024 * 1024; + +# Protocol inspects TCP payload bytes, not TCP/IP headers, to identify the +# application handler Glutton should route to. +public type Protocol = unit { + first: bytes &size=4 { + self.message_length = $$.to_uint(spicy::ByteOrder::Little); + + if ($$ == b"GET " || $$ == b"POST" || $$ == b"HEAD" || $$ == b"OPTI" || $$ == b"CONN") + self.protocol = "http"; + else if ($$ == b"\x03\x00\x00\x2b") + self.protocol = "rdp"; + } + + rest: bytes &eod; + + on %done { + if (self.protocol == "" && self.message_length > 0 && self.message_length <= MaxMongoDBMessageLength && |self.rest| >= 12) { + local op_code = cast(*self.rest.at(8)) | + (cast(*self.rest.at(9)) << 8) | + (cast(*self.rest.at(10)) << 16) | + (cast(*self.rest.at(11)) << 24); + + if (op_code == 1 || op_code == 2001 || op_code == 2002 || op_code == 2004 || op_code == 2005 || op_code == 2006 || op_code == 2007 || op_code == 2012 || op_code == 2013) + self.protocol = "mongodb"; + } + } + + var protocol: string = ""; + var message_length: uint64 = 0; +}; diff --git a/protocols/tcp/http.go b/protocols/tcp/http.go index 221ffbb..6289330 100644 --- a/protocols/tcp/http.go +++ b/protocols/tcp/http.go @@ -180,7 +180,7 @@ func HandleHTTP(ctx context.Context, conn net.Conn, md connection.Metadata, logg } if strings.Contains(req.RequestURI, "/v1.16/version") { - data, err := res.ReadFile("resources/docker_api.json") + data, err := Res.ReadFile("resources/docker_api.json") if err != nil { return fmt.Errorf("failed to read embedded file: %w", err) } diff --git a/protocols/tcp/resources.go b/protocols/tcp/resources.go index 6c68fac..dd922f3 100644 --- a/protocols/tcp/resources.go +++ b/protocols/tcp/resources.go @@ -4,5 +4,5 @@ import "embed" var ( //go:embed resources - res embed.FS + Res embed.FS )