diff --git a/.gitignore b/.gitignore index 48e99efe..4933baab 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,12 @@ ingestors/java/spring/.idea/ ingestors/java/spring/gradle.properties ingestors/java/spring/target/ +ingestors/java/spring-jakarta/.gradle/ +ingestors/java/spring-jakarta/.idea/ +ingestors/java/spring-jakarta/gradle.properties +ingestors/java/spring-jakarta/target/ +ingestors/java/spring-jakarta/out/ + .meta/* ingestors/govxlan/govxlan testing/tmp_tests \ No newline at end of file diff --git a/ingestors/java/spring-jakarta/pom.xml b/ingestors/java/spring-jakarta/pom.xml new file mode 100644 index 00000000..646fb876 --- /dev/null +++ b/ingestors/java/spring-jakarta/pom.xml @@ -0,0 +1,165 @@ + + + 4.0.0 + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.13 + true + + ossrh + https://s01.oss.sonatype.org/ + true + + + + org.apache.maven.plugins + maven-source-plugin + 2.2.1 + + + attach-sources + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.4.1 + + + attach-javadocs + + jar + + + + + + org.apache.maven.plugins + maven-gpg-plugin + 1.5 + + + sign-artifacts + verify + + sign + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.4.1 + + + package + + shade + + + + + + + + true + + + + + + + + jar + + spring + https://www.github.com/metlo-labs/metlo + Java Agent for Metlo + + + MIT License + http://www.opensource.org/licenses/mit-license.php + + + + + Ninad Sinha + ninad@metlo.com + com.metlo + https://www.metlo.com + + + + scm:git:git://github.com/metlo-labs/metlo.git + scm:git:ssh://github.com:metlo-labs/metlo.git + http://github.com/metlo-labs/metlo + + + + 17 + 17 + UTF-8 + + + + + ossrh + https://s01.oss.sonatype.org/content/repositories/snapshots/ + + + + ossrh + https://s01.oss.sonatype.org/content/repositories/releases/ + + + + com.metlo + spring-jakarta + 0.0.1 + + + + com.google.code.gson + gson + 2.9.0 + compile + false + + + org.springframework + spring-context + 6.0.4 + provided + + + org.springframework + spring-web + 6.0.4 + provided + + + jakarta.servlet + jakarta.servlet-api + 5.0.0 + provided + + + jakarta.platform + jakarta.jakartaee-api + 9.0.0 + provided + + + + \ No newline at end of file diff --git a/ingestors/java/spring-jakarta/src/main/java/com/metlo/spring/Metlo.java b/ingestors/java/spring-jakarta/src/main/java/com/metlo/spring/Metlo.java new file mode 100644 index 00000000..ac02b06a --- /dev/null +++ b/ingestors/java/spring-jakarta/src/main/java/com/metlo/spring/Metlo.java @@ -0,0 +1,185 @@ +package com.metlo.spring; + +import com.metlo.spring.utils.ContentCachingResponseWrapperWithHeaderNames; +import com.metlo.spring.utils.RateLimitingRequests; +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.springframework.web.filter.OncePerRequestFilter; +import org.springframework.web.filter.OncePerRequestFilter; +import org.springframework.web.util.ContentCachingRequestWrapper; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.util.*; +import java.util.function.Function; + + +public class Metlo extends OncePerRequestFilter { + private static final int DEFAULT_THREAD_POOL_SIZE = 2; + + private static final int DEFAULT_RPS = 10; + private final static String endpoint = "api/v1/log-request/single"; + + private final Boolean enabled; + + private final RateLimitingRequests req; + + public Metlo(String host, String api_key) { + this(DEFAULT_THREAD_POOL_SIZE, host, api_key, DEFAULT_RPS); + } + + public Metlo(String host, String api_key, Integer rps) { + this(DEFAULT_THREAD_POOL_SIZE, host, api_key, rps); + } + + public Metlo(int pool_size, String host, String api_key, Integer rps) { + + String METLO_ADDR = host; + if (host.charAt(host.length() - 1) == '/') { + METLO_ADDR += endpoint; + } else { + METLO_ADDR += "/" + endpoint; + } + this.req = new RateLimitingRequests(rps, pool_size, METLO_ADDR, api_key); + + String enabled = System.getenv("METLO_ENABLED"); + if (enabled != null) { + this.enabled = Boolean.parseBoolean(enabled); + } else { + this.enabled = true; + } + } + + + @Override + protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain) + throws ServletException, IOException { + + ContentCachingRequestWrapper requestWrapper = new ContentCachingRequestWrapper(request); + ContentCachingResponseWrapperWithHeaderNames responseWrapper = new ContentCachingResponseWrapperWithHeaderNames(response); + + filterChain.doFilter(requestWrapper, responseWrapper); + if (this.enabled) { + String requestBody = getStringValue(requestWrapper.getContentAsByteArray(), + request.getCharacterEncoding()); + String responseBody = getStringValue(responseWrapper.getContentAsByteArray(), + response.getCharacterEncoding()); + + this.req.send(createDataBinding(requestWrapper, responseWrapper, requestBody, responseBody)); + // copyBodyToResponse requires min Spring 4.2.0 + responseWrapper.copyBodyToResponse(); + } + } + + private String getStringValue(byte[] contentAsByteArray, String characterEncoding) { + try { + return new String(contentAsByteArray, 0, contentAsByteArray.length, characterEncoding); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + return ""; + } + + private Map createDataBinding(HttpServletRequest request, ContentCachingResponseWrapperWithHeaderNames response, String request_body, String response_body) { + Map DATA = new HashMap(); + Map REQUEST = new HashMap<>(); + Map REQUEST_URL = new HashMap<>(); + Map RESPONSE = new HashMap<>(); + Map META = new HashMap<>(); + + String host = request.getHeader("host"); + if (host == null) { + host = request.getRemoteHost(); + } + REQUEST_URL.put("host", host); + REQUEST_URL.put("path", request.getRequestURI()); + REQUEST_URL.put("parameters", listParamFormat(request.getParameterMap())); + + REQUEST.put("url", REQUEST_URL); + try { + REQUEST.put("headers", listRequestHeaderFormat(request.getHeaderNames(), request::getHeader)); + } catch (Exception e) { + REQUEST.put("headers", new ArrayList>()); + } + REQUEST.put("body", request_body); + REQUEST.put("method", request.getMethod()); + + RESPONSE.put("url", request.getLocalAddr()); + RESPONSE.put("status", response.getStatus()); + try { + RESPONSE.put("headers", listResponseHeaderFormat(response.getHeaderNames(), response::getHeader)); + } catch (Exception e) { + RESPONSE.put("headers", new ArrayList>()); + } + boolean foundContentType = false; + for (Map header : (ArrayList>) RESPONSE.get("headers")) { + if (header.get("name").equals("Content-Type")) { + header.put("value", response.getResponse().getContentType()); + foundContentType = true; + break; + } + } + if (!foundContentType) { + Map entry = new HashMap(); + entry.put("name", "Content-Type"); + entry.put("value", response.getResponse().getContentType()); + ((ArrayList>) RESPONSE.get("headers")).add(entry); + } + + + RESPONSE.put("body", response_body); + + META.put("source", request.getRemoteAddr()); + META.put("sourcePort", request.getRemotePort()); + META.put("destination", request.getLocalAddr()); + META.put("destinationPort", request.getLocalPort()); + META.put("environment", "production"); + META.put("incoming", "true"); + META.put("metloSource", "java/spring"); + + DATA.put("request", REQUEST); + DATA.put("response", RESPONSE); + DATA.put("meta", META); + + return DATA; + } + + private List> listParamFormat(Map map) { + List> _formatted_params_ = new ArrayList<>(); + for (Map.Entry entry_raw : map.entrySet()) { + HashMap entry = new HashMap(); + entry.put("name", entry_raw.getKey()); + entry.put("value", "[" + String.join(",", entry_raw.getValue()) + "]"); + _formatted_params_.add(entry); + } + return _formatted_params_; + } + + private List> listRequestHeaderFormat(Enumeration headerNames, Function headerValueForName) throws Exception { + List> headers = new ArrayList>(); + while (headerNames.hasMoreElements()) { + String headerName = headerNames.nextElement(); + String headerValue = headerValueForName.apply(headerName); + Map individualHeader = new HashMap(); + individualHeader.put("name", headerName); + individualHeader.put("value", headerValue); + headers.add(individualHeader); + } + return headers; + } + + private List> listResponseHeaderFormat(Collection headerNames, Function headerValueForName) throws Exception { + List> headers = new ArrayList>(); + for (String headerName : headerNames) { + String headerValue = headerValueForName.apply(headerName); + Map individualHeader = new HashMap(); + individualHeader.put("name", headerName); + individualHeader.put("value", headerValue); + headers.add(individualHeader); + } + return headers; + } + +} diff --git a/ingestors/java/spring-jakarta/src/main/java/com/metlo/spring/utils/ContentCachingResponseWrapperWithHeaderNames.java b/ingestors/java/spring-jakarta/src/main/java/com/metlo/spring/utils/ContentCachingResponseWrapperWithHeaderNames.java new file mode 100644 index 00000000..3b9ffbf4 --- /dev/null +++ b/ingestors/java/spring-jakarta/src/main/java/com/metlo/spring/utils/ContentCachingResponseWrapperWithHeaderNames.java @@ -0,0 +1,56 @@ +package com.metlo.spring.utils; + +// Min spring version 4.1.3 + +import jakarta.servlet.http.HttpServletResponse; +import org.springframework.web.util.ContentCachingResponseWrapper; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +public class ContentCachingResponseWrapperWithHeaderNames extends ContentCachingResponseWrapper { + /** + * Here to provide a wrapper to HTTPServletResponse so that we can capture more of the headers. + */ + private final Set headerNames = new HashSet(); + + public ContentCachingResponseWrapperWithHeaderNames(HttpServletResponse delegate) { + super(delegate); + } + + public void addHeader(String name, String value) { + super.addHeader(name, value); + headerNames.add(name); + } + + public void addDateHeader(String name, long date) { + super.addDateHeader(name, date); + headerNames.add(name); + } + + public void addIntHeader(String name, int value) { + super.addIntHeader(name, value); + headerNames.add(name); + } + + public void setHeader(String name, String value) { + super.setHeader(name, value); + headerNames.add(name); + } + + public void setDateHeader(String name, long date) { + super.setDateHeader(name, date); + headerNames.add(name); + } + + public void setIntHeader(String name, int value) { + super.setIntHeader(name, value); + headerNames.add(name); + } + + + public Set getHeaderNames() { + return Collections.unmodifiableSet(headerNames); + } +} diff --git a/ingestors/java/spring-jakarta/src/main/java/com/metlo/spring/utils/RateLimitingRequests.java b/ingestors/java/spring-jakarta/src/main/java/com/metlo/spring/utils/RateLimitingRequests.java new file mode 100644 index 00000000..31cabc0d --- /dev/null +++ b/ingestors/java/spring-jakarta/src/main/java/com/metlo/spring/utils/RateLimitingRequests.java @@ -0,0 +1,98 @@ +package com.metlo.spring.utils; + +// Min Java version 7 (since GSON 2.9) + +import com.google.gson.Gson; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLConnection; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +public class RateLimitingRequests { + private final Integer rps; + private final String host; + private final String key; + private final ThreadPoolExecutor pool; + private final Gson gson; + private List ts; + + public RateLimitingRequests(Integer rps, Integer pool_size, String host, String api_key) { + this.rps = rps; + this.host = host; + this.key = api_key; + this.ts = Collections.synchronizedList(new ArrayList()); + this.gson = new Gson(); + this.pool = new ThreadPoolExecutor(0, pool_size, + 60L, TimeUnit.SECONDS, + new LinkedBlockingQueue()); + } + + private void pushRequest(Map data) throws IOException { + URL url = new URL(this.host); + URLConnection con = url.openConnection(); + HttpURLConnection http = (HttpURLConnection) con; + http.setRequestMethod("POST"); // PUT is another valid option + http.setRequestProperty("Authorization", this.key); + http.setDoOutput(true); + String json = this.gson.toJson(data); + + byte[] out = json.getBytes(StandardCharsets.UTF_8); + int length = out.length; + + + http.setFixedLengthStreamingMode(length); + http.setRequestProperty("Content-Type", "application/json; charset=UTF-8"); + http.connect(); + try (OutputStream os = http.getOutputStream()) { + os.write(out); + } + int code = http.getResponseCode(); + } + + + private synchronized Boolean allow() { + + List tmp_ts = Collections.synchronizedList(new ArrayList()); + Long curr = Instant.now().toEpochMilli(); + this.ts.forEach((Long x) -> { + // We care about requests in the last second only. + if ((curr - x) <= 1000) { + tmp_ts.add(x); + } + }); + + this.ts = tmp_ts; + + if (this.ts.size() < this.rps) { + this.ts.add(curr); + return true; + } + return false; + + } + + public void send(Map data) { + if (this.allow()) { + this.pool.submit(() -> { + try { + pushRequest(data); + } catch (Exception e) { + e.printStackTrace(); + } + }); + } + } + + +}