diff --git a/collector/.gitignore b/collector/.gitignore new file mode 100644 index 00000000..1de56593 --- /dev/null +++ b/collector/.gitignore @@ -0,0 +1 @@ +target \ No newline at end of file diff --git a/collector/Cargo.lock b/collector/Cargo.lock new file mode 100644 index 00000000..6d25adec --- /dev/null +++ b/collector/Cargo.lock @@ -0,0 +1,1523 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" +dependencies = [ + "memchr", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "async-trait" +version = "0.1.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "axum" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8175979259124331c1d7bf6586ee7e0da434155e4b2d48ec2c8386281d8df39" +dependencies = [ + "async-trait", + "axum-core", + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + +[[package]] +name = "base64" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" + +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "time 0.1.45", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "collector" +version = "0.1.0" +dependencies = [ + "axum", + "base64", + "chrono", + "deadpool-postgres", + "deadpool-redis", + "dotenv", + "futures-util", + "hmac", + "lazy_static", + "mime", + "postgres-types", + "redis", + "regex", + "serde", + "serde_json", + "serde_with", + "serde_yaml", + "sha2", + "tokio", + "tokio-postgres", + "url", + "uuid", +] + +[[package]] +name = "combine" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" +dependencies = [ + "bytes", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" + +[[package]] +name = "cpufeatures" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e69e28e9f7f77debdedbaafa2866e1de9ba56df55a8bd7cfc724c25a09987c" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "darling" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0558d22a7b463ed0241e993f76f09f30b126687447751a8638587b864e4b3944" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8bfa2e259f8ee1ce5e97824a3c55ec4404a0d772ca7fa96bf19f0752a046eb" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29a358ff9f12ec09c3e61fef9b5a9902623a695a46a917b07f269bff1445611a" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "deadpool" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "421fe0f90f2ab22016f32a9881be5134fdd71c65298917084b0c7477cbc3856e" +dependencies = [ + "async-trait", + "deadpool-runtime", + "num_cpus", + "retain_mut", + "tokio", +] + +[[package]] +name = "deadpool-postgres" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "836a24a9d49deefe610b8b60c767a7412e9a931d79a89415cd2d2d71630ca8d7" +dependencies = [ + "deadpool", + "log", + "tokio", + "tokio-postgres", +] + +[[package]] +name = "deadpool-redis" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f1760f60ffc6653b4afd924c5792098d8c00d9a3deb6b3d989eac17949dc422" +dependencies = [ + "deadpool", + "redis", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaa37046cc0f6c3cc6090fbdbf73ef0b8ef4cfcc37f6befc0020f63e8cf121e1" +dependencies = [ + "tokio", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "dotenv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" + +[[package]] +name = "equivalent" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88bffebc5d80432c9b140ee17875ff173a8ab62faad5b257da912bd2f6c1c0a1" + +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" + +[[package]] +name = "futures-macro" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" + +[[package]] +name = "futures-task" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" + +[[package]] +name = "futures-util" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +dependencies = [ + "futures-core", + "futures-macro", + "futures-sink", + "futures-task", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" + +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + +[[package]] +name = "hyper" +version = "0.14.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.4.9", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", +] + +[[package]] +name = "itoa" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" + +[[package]] +name = "js-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.146" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" + +[[package]] +name = "lock_api" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" + +[[package]] +name = "matchit" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" + +[[package]] +name = "md-5" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +dependencies = [ + "digest", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +dependencies = [ + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + +[[package]] +name = "percent-encoding" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" + +[[package]] +name = "phf" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c95a7476719eab1e366eaf73d0260af3021184f18177925b07f54b30089ceead" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "postgres-protocol" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b7fa9f396f51dffd61546fd8573ee20592287996568e6175ceb0f8699ad75d" +dependencies = [ + "base64", + "byteorder", + "bytes", + "fallible-iterator", + "hmac", + "md-5", + "memchr", + "rand", + "sha2", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f028f05971fe20f512bcc679e2c10227e57809a3af86a7606304435bc8896cd6" +dependencies = [ + "bytes", + "fallible-iterator", + "postgres-protocol", + "uuid", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "proc-macro2" +version = "1.0.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redis" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ea8c51b5dc1d8e5fd3350ec8167f464ec0995e79f2e90a075b63371500d557f" +dependencies = [ + "async-trait", + "bytes", + "combine", + "futures-util", + "itoa", + "percent-encoding", + "pin-project-lite", + "ryu", + "sha1_smol", + "tokio", + "tokio-util", + "url", +] + +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" + +[[package]] +name = "retain_mut" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4389f1d5789befaf6029ebd9f7dac4af7f7e3d61b69d4f30e2ac02b57e7712b0" + +[[package]] +name = "rustversion" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" + +[[package]] +name = "ryu" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "serde" +version = "1.0.164" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.164" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46266871c240a00b8f503b877622fe33430b3c7d963bdc0f2adc511e54a1eae3" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7f05c1d5476066defcdfacce1f52fc3cae3af1d3089727100c02ae92e5abbe0" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f02d8aa6e3c385bf084924f660ce2a3a6bd333ba55b35e8590b321f35d88513" +dependencies = [ + "base64", + "chrono", + "hex", + "indexmap 1.9.3", + "serde", + "serde_json", + "serde_with_macros", + "time 0.3.22", +] + +[[package]] +name = "serde_with_macros" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc7d5d3932fb12ce722ee5e64dd38c504efba37567f0c402f6ca728c3b8b070" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_yaml" +version = "0.9.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "452e67b9c20c37fa79df53201dc03839651086ed9bbe92b3ca585ca9fdaa7d85" +dependencies = [ + "indexmap 2.0.0", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + +[[package]] +name = "sha2" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + +[[package]] +name = "siphasher" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" + +[[package]] +name = "slab" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" + +[[package]] +name = "socket2" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "socket2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "stringprep" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee348cb74b87454fff4b551cbf727025810a004f88aeacae7f85b87f4e9a1c1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + +[[package]] +name = "syn" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcb8d4cebc40aa517dfb69618fa647a346562e67228e2236ae0042ee6ac14775" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "time" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi", +] + +[[package]] +name = "time" +version = "0.3.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea9e1b3cf1243ae005d9e74085d4d542f3125458f3a81af210d901dcd7411efd" +dependencies = [ + "itoa", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" + +[[package]] +name = "time-macros" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b" +dependencies = [ + "time-core", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.28.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" +dependencies = [ + "autocfg", + "bytes", + "libc", + "mio", + "num_cpus", + "pin-project-lite", + "signal-hook-registry", + "socket2 0.4.9", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-postgres" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e89f6234aa8fd43779746012fcf53603cdb91fdd8399aa0de868c2d56b6dde1" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "socket2 0.5.3", + "tokio", + "tokio-util", +] + +[[package]] +name = "tokio-util" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if", + "log", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + +[[package]] +name = "typenum" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" + +[[package]] +name = "unicode-bidi" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" + +[[package]] +name = "unicode-ident" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unsafe-libyaml" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1865806a559042e51ab5414598446a5871b561d21b6764f2eabb0dd481d880a6" + +[[package]] +name = "url" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "uuid" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa2982af2eec27de306107c027578ff7f423d65f7250e40ce0fea8f45248b81" +dependencies = [ + "getrandom", + "serde", +] + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" diff --git a/collector/Cargo.toml b/collector/Cargo.toml new file mode 100644 index 00000000..0c1ed50e --- /dev/null +++ b/collector/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "collector" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +axum = { version = "0.6.18", features = ["json"] } +base64 = "0.21.2" +chrono = "0.4.26" +deadpool-postgres = "0.10.5" +deadpool-redis = "0.12.0" +dotenv = "0.15.0" +futures-util = "0.3.28" +hmac = "0.12.1" +lazy_static = "1.4.0" +mime = "0.3.17" +postgres-types = "0.2.5" +redis = "0.23.0" +regex = "1.8.4" +serde = { version = "1.0.164", features = ["derive"] } +serde_json = "1.0.99" +serde_with = "3.0.0" +serde_yaml = "0.9.22" +sha2 = "0.10.7" +tokio = { version = "1.28.2", features = ["macros", "rt-multi-thread", "signal"] } +tokio-postgres = { version = "0.7.8", features = ["with-uuid-1"] } +url = "2.4.0" +uuid = { version = "1.3.4", features = ["v4", "serde"] } diff --git a/collector/src/api/log_trace.rs b/collector/src/api/log_trace.rs new file mode 100644 index 00000000..1966400c --- /dev/null +++ b/collector/src/api/log_trace.rs @@ -0,0 +1,290 @@ +use std::collections::{HashMap, HashSet}; + +use axum::{ + extract::{self, Extension, State}, + http::StatusCode, +}; + +use deadpool_redis::redis::cmd; + +use crate::{ + endpoint_tree::get_endpoint_from_tree, + metlo_config::{ + config::{get_config_info, get_mapped_host}, + types::MappedHost, + }, + state::AppState, + types::{ + ApiRequest, ApiUrl, CurrentUser, MetloContext, ProcessTraceRes, ProcessedApiTrace, + QueuedApiTrace, QueuedApiTraceItem, TreeApiEndpoint, + }, + utils::{ + get_valid_path, increment_endpoint_seen_usage_bulk, internal_error, is_graphql_endpoint, + ENDPOINT_CALL_COUNT_HASH, GRAPHQL_SECTIONS, ORG_ENDPOINT_CALL_COUNT, TRACES_QUEUE, + }, +}; + +fn get_content_type(content_type: String) -> String { + match content_type.trim().parse::() { + Ok(m) => m.essence_str().to_owned(), + Err(_) => "*/*".to_owned(), + } +} + +fn get_trace_obj_partial( + trace: ProcessedApiTrace, + valid_path: String, + mapped_host: &MappedHost, +) -> ProcessedApiTrace { + let hosts = match &mapped_host.mapped_host { + Some(e) => (e.to_owned(), Some(trace.request.url.host)), + None => (trace.request.url.host, None), + }; + ProcessedApiTrace { + request: ApiRequest { + method: trace.request.method, + url: ApiUrl { + host: hosts.0, + path: valid_path, + parameters: trace.request.url.parameters, + original_host: hosts.1, + }, + headers: trace.request.headers, + body: trace.request.body, + user: trace.request.user, + }, + response: trace.response, + meta: trace.meta, + processed_trace_data: trace.processed_trace_data.map(|e| ProcessTraceRes { + block: e.block, + attack_detections: e.attack_detections, + sensitive_data_detected: e.sensitive_data_detected, + data_types: e.data_types, + graphql_paths: e.graphql_paths, + request_content_type: get_content_type(e.request_content_type), + response_content_type: get_content_type(e.response_content_type), + request_tags: e.request_tags, + }), + redacted: trace.redacted, + encryption: trace.encryption, + session_meta: trace.session_meta, + analysis_type: trace.analysis_type, + } +} + +fn get_trace_obj_full( + trace: ProcessedApiTrace, + valid_path: String, + status_code: u16, +) -> QueuedApiTrace { + QueuedApiTrace { + path: valid_path, + created_at: chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true), + host: trace.request.url.host, + method: trace.request.method, + request_parameters: trace.request.url.parameters, + request_headers: trace.request.headers, + request_body: trace.request.body, + response_status: status_code, + response_headers: trace + .response + .as_ref() + .map_or(vec![], |e| e.headers.to_owned()), + response_body: trace.response.map(|e| e.body), + meta: trace.meta, + session_meta: trace.session_meta, + processed_trace_data: trace.processed_trace_data.map(|e| ProcessTraceRes { + block: e.block, + attack_detections: e.attack_detections, + sensitive_data_detected: e.sensitive_data_detected, + data_types: e.data_types, + graphql_paths: e.graphql_paths, + request_content_type: get_content_type(e.request_content_type), + response_content_type: get_content_type(e.response_content_type), + request_tags: e.request_tags, + }), + redacted: trace.redacted, + original_host: None, + encryption: trace.encryption, + analysis_type: trace.analysis_type, + } +} + +fn filter_proccessed_data( + attack_detections: HashMap>, + filter: String, +) -> HashMap> { + let mut entry = HashMap::new(); + for item in attack_detections { + let mut split_path = item.0.split('.'); + let token_0 = split_path.next(); + if let Some(token) = token_0 { + let is_graphql_section = GRAPHQL_SECTIONS.contains(&token); + let mut filter_one = filter.to_owned(); + filter_one.push('.'); + let filter_two = token.to_owned() + "." + &filter; + if !is_graphql_section || item.0.contains(&filter_one) || item.0 == filter_two { + entry.insert(item.0, item.1); + } + } + } + entry +} + +fn get_endpoints( + user: &CurrentUser, + traces: &Vec, +) -> Vec> { + let mut endpoints: Vec> = vec![]; + for trace in traces { + endpoints.push(get_endpoint_from_tree(user, trace)); + } + endpoints +} + +fn get_traces(partial_traces: Vec) -> Vec { + let mut graphql_split_traces: Vec = vec![]; + for trace in partial_traces { + let is_graphql = is_graphql_endpoint(&trace.request.url.path); + if let Some(Some(graphql_paths)) = trace + .processed_trace_data + .as_ref() + .map(|e| &e.graphql_paths) + { + for graphql_path in graphql_paths { + let mut split_path = graphql_path.split('.'); + let _token_0 = split_path.next(); + let token_1 = split_path.next(); + let token_2 = split_path.next(); + + if let (Some(tok_1), Some(tok_2)) = (token_1, token_2) { + if tok_1 == "query" || tok_1 == "mutation" || tok_1 == "subscription" { + let filter = tok_1.to_owned() + "." + tok_2; + graphql_split_traces.push(ProcessedApiTrace { + request: ApiRequest { + method: trace.request.method.to_owned(), + url: ApiUrl { + host: trace.request.url.host.to_owned(), + path: trace.request.url.path.to_owned() + "." + &filter, + parameters: trace.request.url.parameters.to_owned(), + original_host: trace.request.url.original_host.to_owned(), + }, + headers: trace.request.headers.to_owned(), + body: trace.request.body.to_owned(), + user: trace.request.user.to_owned(), + }, + response: trace.response.to_owned(), + meta: trace.meta.to_owned(), + processed_trace_data: trace.processed_trace_data.to_owned().map(|e| { + ProcessTraceRes { + block: e.block, + attack_detections: e + .attack_detections + .map(|f| filter_proccessed_data(f, filter)), + sensitive_data_detected: e.sensitive_data_detected, + data_types: e.data_types, + graphql_paths: e.graphql_paths, + request_content_type: e.request_content_type, + response_content_type: e.response_content_type, + request_tags: e.request_tags, + } + }), + redacted: trace.redacted, + encryption: trace.encryption.to_owned(), + session_meta: trace.session_meta.to_owned(), + analysis_type: trace.analysis_type.to_owned(), + }) + } + } + } + } else { + graphql_split_traces.push(trace); + } + } + graphql_split_traces +} + +pub async fn log_trace_batch( + Extension(current_user): Extension, + State(state): State, + extract::Json(traces): extract::Json>, +) -> Result<&'static str, (StatusCode, String)> { + let mut redis_conn = state.redis_pool.get().await.map_err(internal_error)?; + + let redis_queue_length: Result, redis::RedisError> = cmd("LLEN") + .arg(&[TRACES_QUEUE]) + .query_async(&mut redis_conn) + .await; + + let queue_full = match redis_queue_length { + Ok(Some(queue_length)) => queue_length > 1000, + Err(e) => { + println!("Encountered error while checking queue length: {}", e); + true + } + _ => true, + }; + + let mut partial_traces: Vec = vec![]; + let mut full_traces: Vec = vec![]; + + let db_conn = state.db_pool.get().await.map_err(internal_error)?; + let config_info = get_config_info(¤t_user, db_conn, &mut redis_conn).await; + + for trace in traces { + if let Ok(valid_path) = get_valid_path(&trace.request.url.path) { + match trace.analysis_type.as_str() { + "partial" => { + let mapped_host = get_mapped_host( + &config_info, + &trace.request.url.host, + &trace.request.url.path, + ); + if !mapped_host.is_ignored { + partial_traces.push(get_trace_obj_partial(trace, valid_path, &mapped_host)); + } + } + _ if !queue_full => { + if let Some(status_code) = trace.response.as_ref().map(|e| e.status) { + full_traces.push(get_trace_obj_full(trace, valid_path, status_code)); + } + } + _ => (), + } + } + } + + let org_uuid = if current_user.organization_uuid.is_nil() { + None + } else { + Some(current_user.organization_uuid.to_string()) + }; + let mut pipe = redis::pipe(); + for trace in full_traces { + if let Ok(json_str) = serde_json::to_string(&QueuedApiTraceItem { + ctx: MetloContext { + organization_uuid: org_uuid.as_ref().map(|f| f.to_owned()), + }, + version: 2, + trace, + }) { + pipe.cmd("RPUSH").arg(&[TRACES_QUEUE, &json_str]).ignore(); + } + } + if let Err(e) = pipe.query_async::<_, ()>(&mut redis_conn).await { + println!("Encountered error while adding full traces to queue: {}", e); + } + + let graphql_split_traces = get_traces(partial_traces); + let endpoints = get_endpoints(¤t_user, &graphql_split_traces); + + increment_endpoint_seen_usage_bulk( + ¤t_user, + &endpoints, + ENDPOINT_CALL_COUNT_HASH, + ORG_ENDPOINT_CALL_COUNT, + &mut redis_conn, + ) + .await; + Ok("OK") +} diff --git a/collector/src/api/mod.rs b/collector/src/api/mod.rs new file mode 100644 index 00000000..b872bf56 --- /dev/null +++ b/collector/src/api/mod.rs @@ -0,0 +1 @@ +pub mod log_trace; diff --git a/collector/src/endpoint_tree.rs b/collector/src/endpoint_tree.rs new file mode 100644 index 00000000..0b354707 --- /dev/null +++ b/collector/src/endpoint_tree.rs @@ -0,0 +1,260 @@ +use std::cmp::Ordering; + +use serde_json::json; +use tokio_postgres::Row; + +use crate::{ + state::AppState, + types::{CurrentUser, ProcessedApiTrace, TreeApiEndpoint}, + ENDPOINT_TREE, +}; + +use futures_util::{pin_mut, TryStreamExt}; + +fn get_endpoint_from_row(row: Row) -> Result> { + Ok(TreeApiEndpoint { + uuid: row.try_get("uuid")?, + organization_uuid: row.try_get("organizationUuid")?, + path: row.try_get("path")?, + path_regex: row.try_get("pathRegex")?, + host: row.try_get("host")?, + number_params: row.try_get("numberParams")?, + method: row.try_get("method")?, + risk_score: row.try_get("riskScore")?, + is_graph_ql: row.try_get("isGraphQl")?, + user_set: row.try_get("userSet")?, + }) +} + +fn is_parameter(token: &String) -> bool { + if token.is_empty() { + false + } else { + token.starts_with('{') && token.ends_with('}') + } +} + +fn add_path_tokens( + idx: usize, + path_tokens: Vec, + map: &mut serde_json::Map, + endpoint: TreeApiEndpoint, +) { + if idx > path_tokens.len() { + return; + }; + let item = &path_tokens[idx]; + let token = if is_parameter(item) { + "{param}".to_owned() + } else { + item.to_string() + }; + match idx.cmp(&(path_tokens.len() - 1)) { + Ordering::Equal => { + map.insert(token.to_owned(), json!({ "endpoint": endpoint, "children": map.get(&token).and_then(|e| e.get("children"))})); + } + Ordering::Less => { + if map + .get_mut(&token) + .and_then(|e| e.get_mut("children")) + .is_none() + { + map.insert(token.to_owned(), json!({ "endpoint": map.get(&token).and_then(|e| e.get("endpoint")), "children": {}})); + } + if let Some(serde_json::Value::Object(e)) = + map.get_mut(&token).and_then(|e| e.get_mut("children")) + { + add_path_tokens(idx + 1, path_tokens, e, endpoint); + } + } + Ordering::Greater => (), + } +} + +async fn build_endpoint_tree( + state: AppState, +) -> Result> { + let mut tree = json!({ + "children": {} + }); + + let db_conn = state.db_pool.get().await?; + let params: Vec = vec![]; + let stream = db_conn + .query_raw("SELECT * FROM api_endpoint", params) + .await?; + + pin_mut!(stream); + + while let Some(row) = stream.try_next().await? { + match get_endpoint_from_row(row) { + Ok(endpoint) => { + if let Some(serde_json::Value::Object(ptr)) = tree.get_mut("children") { + if ptr + .get_mut(&endpoint.organization_uuid.to_string()) + .is_none() + { + ptr.insert( + endpoint.organization_uuid.to_string(), + json!({ "children": {}}), + ); + } + if let Some(serde_json::Value::Object(ptr_o)) = ptr + .get_mut(&endpoint.organization_uuid.to_string()) + .and_then(|e| e.get_mut("children")) + { + if ptr_o.get_mut(&endpoint.host).is_none() { + ptr_o.insert(endpoint.host.to_owned(), json!({ "children": {}})); + } + if let Some(serde_json::Value::Object(ptr_h)) = ptr_o + .get_mut(&endpoint.host) + .and_then(|e| e.get_mut("children")) + { + if ptr_h.get_mut(&endpoint.method.to_string()).is_none() { + ptr_h.insert(endpoint.method.to_string(), json!({ "children": {}})); + } + if let Some(serde_json::Value::Object(ptr_m)) = ptr_h + .get_mut(&endpoint.method.to_string()) + .and_then(|e| e.get_mut("children")) + { + let path_tokens: Vec = endpoint + .path + .to_owned() + .split('/') + .map(String::from) + .collect(); + let mut start: usize = 0; + if !path_tokens.is_empty() && path_tokens[0].is_empty() { + start = 1 + } + add_path_tokens(start, path_tokens, ptr_m, endpoint); + } + } + } + } + } + Err(e) => println!("Error parsing endpoint row {:?}", e), + } + } + + Ok(tree) +} + +pub async fn refresh_endpoint_tree(state: AppState) { + let res = build_endpoint_tree(state).await; + match res { + Ok(tree) => { + let mut tree_write = ENDPOINT_TREE.write().await; + *tree_write = tree; + drop(tree_write); + } + Err(e) => println!("Error building endpoint tree: {:?}", e), + } +} + +fn find_endpoint_helper( + curr: usize, + path_tokens: &[&str], + num_path_params: i32, + tree: &serde_json::Value, + best_match: &mut Option, +) { + match tree { + serde_json::Value::Object(e) => { + if e.is_empty() { + return; + } + } + serde_json::Value::Null => return, + _ => (), + } + if let Some(m) = best_match { + if num_path_params > m.number_params { + return; + } + } + + let token = path_tokens[curr]; + + if curr == path_tokens.len() - 1 { + let mut matched_endpoint: Option = None; + if tree[token]["endpoint"].is_object() { + if let Some(v) = tree.get(token).and_then(|e| { + e.get("endpoint") + .and_then(|f| serde_json::from_value::(f.to_owned()).ok()) + }) { + matched_endpoint = Some(v); + } + } else if tree["{param}"]["endpoint"].is_object() { + if let Some(v) = tree.get("{param}").and_then(|e| { + e.get("endpoint") + .and_then(|f| serde_json::from_value::(f.to_owned()).ok()) + }) { + matched_endpoint = Some(v); + } + } + if let Some(endpoint) = matched_endpoint { + if best_match + .as_ref() + .map_or(true, |e| endpoint.number_params < e.number_params) + { + *best_match = Some(endpoint); + } + } + return; + } + + if !tree[token].is_null() { + find_endpoint_helper( + curr + 1, + path_tokens, + num_path_params, + &tree[token]["children"], + best_match, + ); + } + if !tree["{param}"].is_null() { + find_endpoint_helper( + curr + 1, + path_tokens, + num_path_params + 1, + &tree["{param}"]["children"], + best_match, + ) + } +} + +fn find_endpoint_recursive( + path_tokens: &[&str], + tree: &serde_json::Value, +) -> Option { + let mut best_match: Option = None; + find_endpoint_helper(0, path_tokens, 0, tree, &mut best_match); + best_match +} + +pub fn get_endpoint_from_tree( + user: &CurrentUser, + trace: &ProcessedApiTrace, +) -> Option { + let tree_read = ENDPOINT_TREE.try_read(); + if let Ok(tree) = tree_read { + let ptr = &tree["children"][user.organization_uuid.to_string()]["children"] + [&trace.request.url.host]["children"][&trace.request.method]["children"]; + if ptr.is_object() { + let path_tokens: Vec<&str> = trace.request.url.path.split('/').collect(); + if path_tokens.len() > 20 { + return None; + } + let mut start: usize = 0; + if !path_tokens.is_empty() && path_tokens[0].is_empty() { + start = 1 + } + let res = find_endpoint_recursive(&path_tokens[start..], ptr); + return res; + } else { + return None; + } + } + None +} diff --git a/collector/src/main.rs b/collector/src/main.rs new file mode 100644 index 00000000..0b3649ec --- /dev/null +++ b/collector/src/main.rs @@ -0,0 +1,77 @@ +mod api; +mod endpoint_tree; +mod metlo_config; +mod metlomiddle; +mod state; +mod types; +mod utils; + +use std::{net::SocketAddr, time::Duration}; + +use axum::{ + middleware, + routing::{get, post}, + Router, +}; +use dotenv::dotenv; + +use endpoint_tree::refresh_endpoint_tree; +use lazy_static::lazy_static; +use serde_json::json; +use tokio::sync::RwLock; +use tokio::time; + +lazy_static! { + pub static ref ENDPOINT_TREE: RwLock = RwLock::new(json!({})); +} + +async fn health() -> &'static str { + "OK" +} + +async fn verify_api_key() -> &'static str { + "OK" +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + dotenv().ok(); + + let app_state = state::AppState::make_app_state().await?; + + let app_state_endpoint_tree = app_state.clone(); + refresh_endpoint_tree(app_state_endpoint_tree.clone()).await; + tokio::task::spawn(async move { + let mut interval = time::interval(Duration::from_secs(300)); + loop { + interval.tick().await; + refresh_endpoint_tree(app_state_endpoint_tree.clone()).await; + } + }); + + let auth_routes = Router::new() + .route("/api/v1/verify", get(verify_api_key)) + .route("/api/v2/verify", get(verify_api_key)) + .route( + "/api/v2/log-request/batch", + post(api::log_trace::log_trace_batch), + ) + .route_layer(middleware::from_fn_with_state( + app_state.clone(), + metlomiddle::auth::auth, + )); + let no_auth_routes = Router::new().route("/api/v2", get(health)); + + let app = Router::new() + .merge(auth_routes) + .merge(no_auth_routes) + .with_state(app_state); + + let addr = SocketAddr::from(([127, 0, 0, 1], 8081)); + axum::Server::bind(&addr) + .serve(app.into_make_service()) + .await + .unwrap(); + + Ok(()) +} diff --git a/collector/src/metlo_config/config.rs b/collector/src/metlo_config/config.rs new file mode 100644 index 00000000..8c2ad035 --- /dev/null +++ b/collector/src/metlo_config/config.rs @@ -0,0 +1,159 @@ +use deadpool_postgres::Object; +use deadpool_redis::Connection; +use redis::cmd; +use regex::Regex; + +use crate::types::CurrentUser; + +use super::types::{ + ConfigInfo, HostMappingCompiled, MappedHost, MetloConfigType, PathIgnoreListCompiled, +}; + +async fn get_metlo_config_processed(user: &CurrentUser, db_conn: Object) -> String { + let res = db_conn + .query_one( + "SELECT * FROM metlo_config WHERE \"organizationUuid\" = $1", + &[&user.organization_uuid], + ) + .await; + match res { + Ok(row) => row.get("configString"), + Err(_) => "".to_owned(), + } +} + +async fn get_metlo_config_processed_cached( + user: &CurrentUser, + db_conn: Object, + redis_conn: &mut Connection, +) -> MetloConfigType { + let default = MetloConfigType { + host_map: None, + host_block_list: None, + path_block_list: None, + ignored_detections: None, + }; + let mut redis_key = "_cachedMetloConfig".to_owned(); + if !&user.organization_uuid.is_nil() { + redis_key.insert_str(0, user.organization_uuid.to_string().as_str()); + } + let redis_key_str = redis_key.as_str(); + let redis_metlo_config_str: Option = cmd("GET") + .arg(&[redis_key_str]) + .query_async(redis_conn) + .await + .ok(); + + match redis_metlo_config_str { + Some(config) => serde_yaml::from_str::(&config).unwrap_or(default), + None => { + let config_string = get_metlo_config_processed(user, db_conn).await; + if let Err(e) = redis::pipe() + .cmd("SET") + .arg(&[redis_key_str, &config_string]) + .ignore() + .cmd("EXPIRE") + .arg(&[redis_key_str, "60"]) + .ignore() + .query_async::<_, ()>(redis_conn) + .await + { + println!( + "Encountered error while saving Metlo Config in cache: {}", + e + ); + } + serde_yaml::from_str::(&config_string).unwrap_or(default) + } + } +} + +pub async fn get_config_info( + user: &CurrentUser, + db_conn: Object, + redis_conn: &mut Connection, +) -> ConfigInfo { + let res = get_metlo_config_processed_cached(user, db_conn, redis_conn).await; + ConfigInfo { + compiled_host_map: res.host_map.map(|e| { + e.iter() + .filter_map(|h| match Regex::new(&h.pattern) { + Ok(r) => Some(HostMappingCompiled { + host: h.host.to_owned(), + pattern: r, + }), + Err(_) => None, + }) + .collect() + }), + compiled_host_ignore_list: res.host_block_list.map(|e| { + e.iter() + .filter_map(|h| match Regex::new(h) { + Ok(r) => Some(r), + Err(_) => None, + }) + .collect() + }), + compiled_path_ignore_list: res.path_block_list.map(|e| { + e.iter() + .filter_map(|h| match Regex::new(&h.host) { + Ok(r) => Some(PathIgnoreListCompiled { + host: r, + paths: h + .paths + .iter() + .filter_map(|p| match Regex::new(p) { + Ok(reg) => Some(reg), + Err(_) => None, + }) + .collect(), + }), + Err(_) => None, + }) + .collect() + }), + ignored_detections: res.ignored_detections, + } +} + +pub fn get_mapped_host(config_info: &ConfigInfo, trace_host: &str, trace_path: &str) -> MappedHost { + let mut res = MappedHost { + mapped_host: None, + is_ignored: false, + }; + + if let Some(map) = config_info + .compiled_host_map + .as_ref() + .and_then(|e| e.iter().find(|&h| h.pattern.is_match(trace_host))) + { + res.mapped_host = Some(map.host.to_owned()); + } + + if let Some(host_ignore_list) = &config_info.compiled_host_ignore_list { + if !host_ignore_list.is_empty() { + for ignored_host in host_ignore_list { + if ignored_host.is_match(trace_host) { + res.is_ignored = true; + break; + } + } + } + } + + if !res.is_ignored { + if let Some(path_ignore_list) = &config_info.compiled_path_ignore_list { + for item in path_ignore_list { + if item.host.is_match(trace_host) { + for ignored_path in &item.paths { + if ignored_path.is_match(trace_path) { + res.is_ignored = true; + break; + } + } + } + } + } + } + res +} diff --git a/collector/src/metlo_config/mod.rs b/collector/src/metlo_config/mod.rs new file mode 100644 index 00000000..c6635022 --- /dev/null +++ b/collector/src/metlo_config/mod.rs @@ -0,0 +1,2 @@ +pub mod config; +pub mod types; diff --git a/collector/src/metlo_config/types.rs b/collector/src/metlo_config/types.rs new file mode 100644 index 00000000..31cf6d78 --- /dev/null +++ b/collector/src/metlo_config/types.rs @@ -0,0 +1,59 @@ +use std::collections::HashMap; + +use regex::Regex; +use serde::Deserialize; + +pub struct MappedHost { + pub mapped_host: Option, + pub is_ignored: bool, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct HostMapping { + pub host: String, + pub pattern: String, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PathIgnoreList { + pub host: String, + pub paths: Vec, +} + +pub struct HostMappingCompiled { + pub host: String, + pub pattern: Regex, +} + +pub struct PathIgnoreListCompiled { + pub host: Regex, + pub paths: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct IgnoredDetection { + pub id: Option, + pub host: Option, + pub path: Option, + pub method: Option, + pub ignored_paths: HashMap>, +} + +pub struct ConfigInfo { + pub compiled_host_map: Option>, + pub compiled_host_ignore_list: Option>, + pub compiled_path_ignore_list: Option>, + pub ignored_detections: Option>, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MetloConfigType { + pub host_map: Option>, + pub host_block_list: Option>, + pub path_block_list: Option>, + pub ignored_detections: Option>, +} diff --git a/collector/src/metlomiddle/auth.rs b/collector/src/metlomiddle/auth.rs new file mode 100644 index 00000000..4f214ff0 --- /dev/null +++ b/collector/src/metlomiddle/auth.rs @@ -0,0 +1,92 @@ +use std::str::FromStr; + +use axum::{ + extract::State, + http::{self, Request, StatusCode}, + middleware::Next, + response::Response, +}; +use base64::{engine::general_purpose, Engine as _}; +use deadpool_redis::redis::{self, cmd}; +use hmac::{Hmac, Mac}; +use sha2::Sha512; +use uuid::Uuid; + +use crate::{state::AppState, types::CurrentUser, utils::internal_error}; + +type HmacSha512 = Hmac; + +pub async fn auth( + State(state): State, + mut req: Request, + next: Next, +) -> Result { + // Get Auth Header + let auth_header = req + .headers() + .get(http::header::AUTHORIZATION) + .and_then(|header| header.to_str().ok()); + let auth_header = if let Some(auth_header) = auth_header { + auth_header + } else { + return Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_string())); + }; + + // Generate Hash + let mut mac = + HmacSha512::new_from_slice(state.encryption_key.as_bytes()).map_err(internal_error)?; + mac.update(auth_header.as_bytes()); + let hash_bytes = mac.finalize().into_bytes(); + let api_key_hash = general_purpose::STANDARD.encode(hash_bytes); + + // Check Redis for API Key Hash + let mut redis_conn = state.redis_pool.get().await.map_err(internal_error)?; + let mut redis_key = "collector_cached_api_key_".to_string(); + redis_key.push_str(&api_key_hash); + let redis_key_str = redis_key.as_str(); + let redis_org_uuid: Option = cmd("GET") + .arg(&[redis_key_str]) + .query_async(&mut redis_conn) + .await + .map_err(internal_error)?; + if let Some(unwrapped_org_uuid_str) = redis_org_uuid { + if let Ok(org_uuid_res) = Uuid::from_str(unwrapped_org_uuid_str.as_str()) { + req.extensions_mut().insert(CurrentUser { + user_uuid: None, + organization_uuid: org_uuid_res, + }); + return Ok(next.run(req).await); + } + } + + // Check DB for API Key Hash + let db_conn = state.db_pool.get().await.map_err(internal_error)?; + let res = db_conn + .query( + "SELECT * FROM api_key WHERE \"apiKeyHash\" = $1::TEXT", + &[&api_key_hash], + ) + .await + .map_err(internal_error)?; + let org_uuid: Option = res.get(0).map(|e| e.get("organizationUuid")); + + if let Some(unwrapped_org_uuid) = org_uuid { + req.extensions_mut().insert(CurrentUser { + user_uuid: None, + organization_uuid: unwrapped_org_uuid, + }); + redis::pipe() + .cmd("SET") + .arg(&[redis_key_str, unwrapped_org_uuid.to_string().as_str()]) + .ignore() + .cmd("EXPIRE") + .arg(&[redis_key_str, "30"]) + .ignore() + .query_async::<_, ()>(&mut redis_conn) + .await + .map_err(internal_error)?; + Ok(next.run(req).await) + } else { + Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_string())) + } +} diff --git a/collector/src/metlomiddle/mod.rs b/collector/src/metlomiddle/mod.rs new file mode 100644 index 00000000..0e4a05d5 --- /dev/null +++ b/collector/src/metlomiddle/mod.rs @@ -0,0 +1 @@ +pub mod auth; diff --git a/collector/src/state.rs b/collector/src/state.rs new file mode 100644 index 00000000..ccb04a8e --- /dev/null +++ b/collector/src/state.rs @@ -0,0 +1,49 @@ +use std::env; + +use deadpool_postgres::{Config, ManagerConfig, Pool as PostgresPool, RecyclingMethod}; +use deadpool_redis::{Config as RedisConfig, Pool as RedisPool, Runtime}; +use tokio_postgres::NoTls; +use url::Url; + +#[derive(Clone)] +pub struct AppState { + pub db_pool: PostgresPool, + pub redis_pool: RedisPool, + pub encryption_key: String, +} + +impl AppState { + pub async fn make_app_state() -> Result> { + let db_conn_string = + env::var("DB_URL").map_err(|e| format!("Error getting DB_URL: {}", e))?; + let redis_url = + env::var("REDIS_URL").map_err(|e| format!("Error getting REDIS_URL: {}", e))?; + let encryption_key = env::var("ENCRYPTION_KEY") + .map_err(|e| format!("Error getting ENCRYPTION_KEY: {}", e))?; + + let db_url = Url::parse(&db_conn_string)?; + + let mut db_config = Config::new(); + db_config.user = Some(db_url.username().to_string()).filter(|e| !e.is_empty()); + db_config.dbname = Some(db_url.path().to_string()) + .map(|e| e.trim_start_matches('/').to_string()) + .filter(|e| !e.is_empty()); + db_config.host = db_url.host().map(|e| e.to_string()); + db_config.port = db_url.port(); + db_config.manager = Some(ManagerConfig { + recycling_method: RecyclingMethod::Fast, + }); + let db_pool = db_config.create_pool(None, NoTls).unwrap(); + db_pool.resize(16); + + let redis_config = RedisConfig::from_url(redis_url); + let redis_pool = redis_config.create_pool(Some(Runtime::Tokio1)).unwrap(); + redis_pool.resize(4); + + Ok(AppState { + db_pool, + redis_pool, + encryption_key, + }) + } +} diff --git a/collector/src/types.rs b/collector/src/types.rs new file mode 100644 index 00000000..8eb2f735 --- /dev/null +++ b/collector/src/types.rs @@ -0,0 +1,299 @@ +use std::{ + collections::{HashMap, HashSet}, + error::Error, + fmt, +}; + +use postgres_types::Type; +use serde::{Deserialize, Serialize}; +use serde_with::skip_serializing_none; +use tokio_postgres::types::FromSql; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum RestMethod { + Get, + Head, + Post, + Put, + Patch, + Delete, + Connect, + Options, + Trace, +} + +impl FromSql<'_> for RestMethod { + fn from_sql(_sql_type: &Type, value: &[u8]) -> Result> { + match value { + b"GET" => Ok(RestMethod::Get), + b"HEAD" => Ok(RestMethod::Head), + b"POST" => Ok(RestMethod::Post), + b"PUT" => Ok(RestMethod::Put), + b"PATCH" => Ok(RestMethod::Patch), + b"DELETE" => Ok(RestMethod::Delete), + b"CONNECT" => Ok(RestMethod::Connect), + b"OPTIONS" => Ok(RestMethod::Options), + b"TRACE" => Ok(RestMethod::Trace), + _ => Err("Couldn't parse rest method from query result".into()), + } + } + + fn accepts(sql_type: &Type) -> bool { + sql_type.name() == "rest_method_enum" + } +} + +impl Serialize for RestMethod { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(match self { + RestMethod::Get => "GET", + RestMethod::Head => "HEAD", + RestMethod::Post => "POST", + RestMethod::Put => "PUT", + RestMethod::Patch => "PATCH", + RestMethod::Delete => "DELETE", + RestMethod::Connect => "CONNECT", + RestMethod::Options => "OPTIONS", + RestMethod::Trace => "TRACE", + }) + } +} + +impl<'de> Deserialize<'de> for RestMethod { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + Ok(match s.as_str() { + "GET" => RestMethod::Get, + "HEAD" => RestMethod::Head, + "POST" => RestMethod::Post, + "PUT" => RestMethod::Put, + "PATCH" => RestMethod::Patch, + "DELETE" => RestMethod::Delete, + "CONNECT" => RestMethod::Connect, + "OPTIONS" => RestMethod::Options, + "TRACE" => RestMethod::Trace, + _ => RestMethod::Get, + }) + } +} + +impl fmt::Display for RestMethod { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + RestMethod::Get => write!(f, "GET"), + RestMethod::Head => write!(f, "HEAD"), + RestMethod::Post => write!(f, "POST"), + RestMethod::Put => write!(f, "PUT"), + RestMethod::Patch => write!(f, "PATCH"), + RestMethod::Delete => write!(f, "DELETE"), + RestMethod::Connect => write!(f, "CONNECT"), + RestMethod::Options => write!(f, "OPTIONS"), + RestMethod::Trace => write!(f, "TRACE"), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum RiskScore { + High, + Medium, + Low, + None, +} + +impl FromSql<'_> for RiskScore { + fn from_sql(_sql_type: &Type, value: &[u8]) -> Result> { + match value { + b"high" => Ok(RiskScore::High), + b"medium" => Ok(RiskScore::Medium), + b"low" => Ok(RiskScore::Low), + b"none" => Ok(RiskScore::None), + _ => Err("Couldn't parse risk score from query result".into()), + } + } + + fn accepts(sql_type: &Type) -> bool { + sql_type.name() == "riskscore_enum" + } +} + +impl Serialize for RiskScore { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(match self { + RiskScore::High => "high", + RiskScore::Medium => "medium", + RiskScore::Low => "low", + RiskScore::None => "none", + }) + } +} + +impl<'de> Deserialize<'de> for RiskScore { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + Ok(match s.as_str() { + "high" => RiskScore::High, + "medium" => RiskScore::Medium, + "low" => RiskScore::Low, + "none" => RiskScore::None, + _ => RiskScore::None, + }) + } +} + +#[derive(Clone, Debug)] +pub struct CurrentUser { + pub user_uuid: Option, + pub organization_uuid: uuid::Uuid, +} + +#[derive(Deserialize, Debug, Clone, Serialize)] +pub struct KeyVal { + pub name: String, + pub value: String, +} + +#[derive(Deserialize, Debug, Clone, Serialize)] +pub struct ApiUrl { + pub host: String, + pub path: String, + pub parameters: Vec, + pub original_host: Option, +} + +#[derive(Deserialize, Debug, Clone, Serialize)] +pub struct ApiRequest { + pub method: String, + pub url: ApiUrl, + pub headers: Vec, + pub body: String, + pub user: Option, +} + +#[derive(Deserialize, Debug, Clone, Serialize)] +pub struct ApiResponse { + pub status: u16, + pub headers: Vec, + pub body: String, +} + +#[derive(Deserialize, Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ApiMeta { + pub environment: String, + pub incoming: bool, + pub source: String, + pub source_port: u16, + pub destination: String, + pub destination_port: u16, + pub original_source: Option, +} + +#[skip_serializing_none] +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ProcessTraceRes { + pub block: bool, + pub attack_detections: Option>>, + pub sensitive_data_detected: Option>>, + pub data_types: Option>>, + pub graphql_paths: Option>, + pub request_content_type: String, + pub response_content_type: String, + pub request_tags: Option>, +} + +#[derive(Debug, Deserialize, Serialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct SessionMeta { + pub authentication_provided: Option, + pub authentication_successful: Option, + pub auth_type: Option, + pub unique_session_key: Option, + pub user: Option, + pub user_agent: Option, +} + +#[derive(Debug, Deserialize, Serialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Encryption { + pub key: String, + pub generated_ivs: HashMap>, +} + +#[skip_serializing_none] +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ProcessedApiTrace { + pub request: ApiRequest, + pub response: Option, + pub meta: Option, + pub processed_trace_data: Option, + pub redacted: bool, + pub encryption: Option, + pub session_meta: Option, + pub analysis_type: String, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct QueuedApiTrace { + pub path: String, + pub created_at: String, + pub host: String, + pub method: String, + pub request_parameters: Vec, + pub request_headers: Vec, + pub request_body: String, + pub response_status: u16, + pub response_headers: Vec, + pub response_body: Option, + pub meta: Option, + pub session_meta: Option, + pub processed_trace_data: Option, + pub redacted: bool, + pub original_host: Option, + pub encryption: Option, + pub analysis_type: String, +} + +#[skip_serializing_none] +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct MetloContext { + pub organization_uuid: Option, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct QueuedApiTraceItem { + pub ctx: MetloContext, + pub version: u8, + pub trace: QueuedApiTrace, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TreeApiEndpoint { + pub uuid: uuid::Uuid, + pub organization_uuid: uuid::Uuid, + pub path: String, + pub path_regex: String, + pub host: String, + pub number_params: i32, + pub method: RestMethod, + pub risk_score: RiskScore, + pub is_graph_ql: bool, + pub user_set: bool, +} diff --git a/collector/src/utils.rs b/collector/src/utils.rs new file mode 100644 index 00000000..045c158a --- /dev/null +++ b/collector/src/utils.rs @@ -0,0 +1,105 @@ +use std::time::{SystemTime, UNIX_EPOCH}; + +use axum::http::StatusCode; +use deadpool_redis::Connection; + +use crate::types::{CurrentUser, TreeApiEndpoint}; +pub fn internal_error(err: E) -> (StatusCode, String) +where + E: std::error::Error, +{ + (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) +} + +pub const GRAPHQL_SECTIONS: [&str; 3] = ["reqBody", "reqQuery", "resBody"]; +pub const TRACES_QUEUE: &str = "traces_queue"; +pub const ENDPOINT_CALL_COUNT_HASH: &str = "endpoints_call_count"; +pub const ORG_ENDPOINT_CALL_COUNT: &str = "org_endpoints_call_count"; +const GRAPHQL_PATHS: [&str; 1] = ["/graphql"]; +const USAGE_GRANULARITY: u128 = 1000 * 60; + +pub fn get_valid_path(path: &str) -> Result> { + if path.is_empty() { + return Err("No path provided".into()); + } + if !path.starts_with('/') { + return Err("Path does not start with a leading slash.".into()); + } + + if path == "/" { + return Ok("/".to_owned()); + } + + let tokens = path.split('/'); + let mut empty_tokens: u16 = 0; + let mut valid_path_tokens: Vec<&str> = vec![]; + + for token in tokens { + if token.is_empty() { + empty_tokens += 1; + } else { + valid_path_tokens.push(token) + } + } + + if empty_tokens > 2 { + return Err("Too many trailing or leading slashes in path.".into()); + } + + let mut valid_path = valid_path_tokens.join("/"); + valid_path.insert(0, '/'); + + Ok(valid_path) +} + +pub fn is_graphql_endpoint(path: &str) -> bool { + let trimmed = path.trim_end_matches('/'); + GRAPHQL_PATHS.iter().any(|e| trimmed.ends_with(e)) +} + +pub async fn increment_endpoint_seen_usage_bulk( + user: &CurrentUser, + endpoints: &[Option], + endpoint_call_count_key: &str, + org_call_count_key: &str, + redis_conn: &mut Connection, +) { + let mut endpoint_call_key = "_".to_owned() + endpoint_call_count_key; + let mut org_call_key = "_".to_owned() + org_call_count_key; + if !user.organization_uuid.is_nil() { + let org_str = user.organization_uuid.to_string(); + endpoint_call_key.insert_str(0, org_str.as_str()); + org_call_key.insert_str(0, org_str.as_str()); + } + let curr_time = SystemTime::now().duration_since(UNIX_EPOCH); + if let Ok(duration) = curr_time { + let curr_time_millis = duration.as_millis(); + let time_slot = curr_time_millis - (curr_time_millis % USAGE_GRANULARITY); + org_call_key.push('_'); + org_call_key.push_str(time_slot.to_string().as_str()); + let org_call_key_str = org_call_key.as_str(); + let endpoint_call_key_str = endpoint_call_key.as_str(); + let mut pipe = redis::pipe(); + let mut endpoint_count = 0; + for endpoint in endpoints.iter().flatten() { + endpoint_count += 1; + pipe.cmd("HINCRBY") + .arg(&[ + endpoint_call_key_str, + endpoint.uuid.to_string().as_str(), + "1", + ]) + .ignore(); + } + let pipe = pipe + .cmd("INCRBY") + .arg(&[org_call_key_str, endpoint_count.to_string().as_str()]) + .ignore() + .cmd("EXPIRE") + .arg(&[org_call_key_str, "120"]) + .ignore(); + if let Err(e) = pipe.query_async::<_, ()>(redis_conn).await { + println!("Encountered error while incrementing endpoint usage: {}", e); + } + } +}