Add SDA Workflow

.gitignore

@@ -188,3 +188,11 @@ cache
 # IntelliJ IDEA files
 .idea
 *.iml
+
+# Local config
+config/config.json
+config/redis
+
+# Benchmark Results
+results/
+experiments.json

.vscode/launch.json

@@ -0,0 +1,84 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "SeBS Benchmark Workflow 610",
+            "type": "python",
+            "request": "launch",
+            "program": "${workspaceFolder}/sebs.py",
+            "cwd": "${workspaceFolder}",
+            "console": "integratedTerminal",
+            "args": [
+                "benchmark",
+                "workflow",
+                "610.gen",
+                "test",
+                "--config",
+                "config/config.json",
+                "--deployment",
+                "aws",
+                "--container-deployment",
+                "--trigger",
+                "library",
+                "--repetitions",
+                "1"
+            ],
+            "justMyCode": true,
+            "pythonPath": "${workspaceFolder}/python-venv/bin/python",
+            "envFile": "${workspaceFolder}/.env"
+        },
+        {
+            "name": "SeBS SDA Workflow Test",
+            "type": "python",
+            "request": "launch",
+            "program": "${workspaceFolder}/sebs.py",
+            "cwd": "${workspaceFolder}",
+            "console": "integratedTerminal",
+            "args": [
+                "benchmark",
+                "workflow",
+                "6300.sda-workflow",
+                "test",
+                "--config",
+                "config/config.json",
+                "--deployment",
+                "aws",
+                "--trigger",
+                "library",
+                "--repetitions",
+                "1"
+            ],
+            "justMyCode": true,
+            "pythonPath": "${workspaceFolder}/python-venv/bin/python",
+            "envFile": "${workspaceFolder}/.env"
+        },
+        {
+            "name": "SeBS SDA Workflow",
+            "type": "python",
+            "request": "launch",
+            "program": "${workspaceFolder}/sebs.py",
+            "cwd": "${workspaceFolder}",
+            "console": "integratedTerminal",
+            "args": [
+                "benchmark",
+                "workflow",
+                "6300.sda-workflow",
+                "large",
+                "--config",
+                "config/config.json",
+                "--deployment",
+                "aws",
+                "--trigger",
+                "library",
+                "--repetitions",
+                "1"
+            ],
+            "justMyCode": true,
+            "pythonPath": "${workspaceFolder}/python-venv/bin/python",
+            "envFile": "${workspaceFolder}/.env"
+        }
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "python-envs.pythonProjects": []
+}

benchmarks/600.workflows/610.gen/config.json

@@ -1,5 +1,6 @@
 {
   "timeout": 120,
   "memory": 128,
-  "languages": ["python"]
+  "languages": ["python"],
+  "modules": []
 }

benchmarks/600.workflows/610.gen/definition.json

@@ -37,10 +37,10 @@
         "map_astros": {
             "type": "map",
             "array": "astros.people",
-            "root": "map_astros",
+            "root": "map",
             "next": "process_astros",
             "states": {
-              "map_astros": {
+              "map": {
                 "type": "task",
                 "func_name": "map_astros"
               }

benchmarks/600.workflows/610.gen/input.py

@@ -1,5 +1,5 @@
 def buckets_count():
     return (0, 0)
 
-def generate_input(data_dir, size, input_buckets, output_buckets, upload_func):
+def generate_input(data_dir, size, benchmarks_bucket,input_buckets, output_buckets, upload_func, nosql_func):
     return dict()

benchmarks/600.workflows/610.gen/python/map_astros.py

@@ -1,7 +1,6 @@
 def handler(elem):
-    name = elem["name"]
-    fn, ln = name.split(" ")
-    name = " ".join([ln, fn])
-    elem["name_rev"] = name
-
+    full_name:str = elem["name"]
+    names = full_name.split(" ")
+    names.reverse()
+    elem["name_rev"] = " ".join(names)
     return elem

benchmarks/600.workflows/610.gen/python/requirements.txt

@@ -0,0 +1 @@
+requests

benchmarks/600.workflows/6300.sda-workflow/.gitignore

@@ -0,0 +1,2 @@
+cfg/
+dev/

benchmarks/600.workflows/6300.sda-workflow/config.json

@@ -0,0 +1,7 @@
+{
+  "timeout": 500,
+  "memory": 128,
+  "languages": ["python"],
+  "modules": ["storage"],
+  "container-image": "logru/sda-no-db:latest"
+}

benchmarks/600.workflows/6300.sda-workflow/definition.json

@@ -0,0 +1,70 @@
+{
+    "root": "clearDB",
+    "states": {
+        "clearDB":{
+            "type": "task",
+            "func_name": "clearDB",
+            "next": "split"
+        },
+        "split": {
+            "type": "task",
+            "func_name": "split",
+            "next": "filter-map"
+        },
+        "filter-map":{
+            "type": "map",
+            "root": "filter",
+            "array": "filter_workloads",
+            "next": "prepare-neighbors",
+            "states": {
+                "filter": {
+                    "type": "task",
+                    "func_name": "filter"
+                }
+            }
+        },
+        "prepare-neighbors": {
+            "type": "task",
+            "func_name": "pre_neighbors",
+            "next": "neighbors-map"
+        },
+        "neighbors-map":{
+            "type": "map",
+            "root": "neighbors",
+            "array": "neighbors_workloads",
+            "next": "components",
+            "states": {
+                "neighbors": {
+                    "type": "task",
+                    "func_name": "neighbors"
+                }
+            }
+        },
+        "components": {
+            "type": "task",
+            "func_name": "components",
+            "next": "cluster-analyze-map"
+        },
+        "cluster-analyze-map":{
+            "type": "map",
+            "root": "clustering",
+            "array": "cluster_workloads",
+            "next": "merge-results",
+            "states": {
+                "clustering": {
+                    "type": "task",
+                    "func_name": "clustering",
+                    "next": "analysis"
+                },
+                "analysis": {
+                    "type": "task",
+                    "func_name": "analysis"
+                }
+            }
+        },
+        "merge-results": {
+            "type": "task",
+            "func_name": "merge"
+        }
+    }
+}

benchmarks/600.workflows/6300.sda-workflow/dev/Dockerfile

@@ -0,0 +1,23 @@
+# Define custom base image
+ARG BASE_IMAGE="python:3.12"
+
+FROM ${BASE_IMAGE} AS build-image
+RUN apt-get update && apt-get install -y python3-pip
+
+# Include global arg in this stage of the build
+ARG LAMBDA_DIR="/lambda"
+
+# Copy function code
+RUN mkdir -p ${LAMBDA_DIR}
+RUN mkdir -p ${LAMBDA_DIR}/function
+# Install the function's dependencies
+RUN pip install \
+    --target ${LAMBDA_DIR} \
+        awslambdaric
+
+COPY . ${LAMBDA_DIR}/function
+WORKDIR ${LAMBDA_DIR}
+# Set runtime interface client as default command for the container runtime
+ENTRYPOINT [ "/usr/bin/python", "-m", "awslambdaric" ]
+# Pass the name of the function handler as an argument to the runtime
+CMD [ "function/handler.handler" ]

benchmarks/600.workflows/6300.sda-workflow/dev/README.md

@@ -0,0 +1,31 @@
+# Docker Commands:
+### Build Image
+```
+docker build  \
+--build-arg BASE_IMAGE=logru/sda:latest \
+-t logru/sda-sebs-aws \
+-f benchmarks/600.workflows/6300.sda-workflow/dev/Dockerfile \
+6300.sda-workflow_code/python/3.8/x64/package/
+```
+### Test Run
+[Custom Image Tutorial](https://docs.aws.amazon.com/lambda/latest/dg/python-image.html#python-image-clients)
+#### Start Container
+```
+docker run \
+--platform linux/amd64 -d \
+-v ~/.aws-lambda-rie:/aws-lambda \
+-p 9000:8080 \
+-e AWS_LAMBDA_FUNCTION_NAME=SDA___filter \
+--entrypoint /aws-lambda/aws-lambda-rie \
+--name sda-aws-container \
+logru/sda-sebs-aws:latest \
+/usr/bin/python -m awslambdaric function/handler.handler 
+``` 
+#### Debug Container
+```
+docker exec -it sda-aws-container /bin/bash
+```
+#### Trigger Requests
+```
+curl "http://localhost:9000/2015-03-31/functions/function/invocations" -d '{"payload":{},"request_id":"1"}'
+```

benchmarks/600.workflows/6300.sda-workflow/dev/data/sda-workflow-local.json

@@ -0,0 +1,41 @@
+{
+    "analysis-output-stem": "Analysis",
+    "binary-filters": [],
+    "centrality-measures": [
+        {
+            "name": "DegreeCentrality"
+        },
+        {
+            "name": "MeanLocalSignificance"
+        },
+        {
+            "name": "SmallerNeighboursRatio"
+        }
+    ],
+    "cleanup": true,
+    "concurrent-runs": true,
+    "contraction-output-stem": "Contraction",
+    "contraction-predicates": [
+        {
+            "distance": 500.0,
+            "name": "DistanceBiPredicate"
+        }
+    ],
+    "executor": {
+        "cwl-directory": "/home/lolo/Projects/fishnet/cwl/sda",
+        "flags": "--quiet --no-container",
+        "name": "CWLTOOL"
+    },
+    "hardware-concurrency": 0,
+    "last-job-type": "FILTER",
+    "maxDistanceMeters": 3000.0,
+    "maxNeighbours": 5,
+    "memgraph-host": "localhost",
+    "memgraph-port": 7687,
+    "merge-workers": 2,
+    "neighbouring-files-predicate": "WSF",
+    "neighbouring-predicates": [],
+    "splits": 0,
+    "unary-filters": [],
+    "visualize-edges": true
+}

benchmarks/600.workflows/6300.sda-workflow/dev/debug.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+if [ -n "$(docker ps -aq -f name=^sda-aws-container$)" ]; then
+    echo "Removing existing container sda-aws-container..."
+    docker rm -f sda-aws-container >/dev/null 2>&1 || true
+fi
+WORKFLOW_NAME="SDA"
+FUNCTION="filter"
+docker run --platform linux/amd64 -d -v ~/.aws-lambda-rie:/aws-lambda -v /home/lolo/Projects/serverless-benchmarks/benchmarks/600.workflows/6300.sda-workflow/dev/data:/data -p 9000:8080 -e AWS_LAMBDA_FUNCTION_NAME=${WORKFLOW_NAME}___${FUNCTION} --entrypoint /aws-lambda/aws-lambda-rie --name sda-aws-container logru/sda-sebs-aws:latest /usr/bin/python -m awslambdaric function/handler.handler 
+docker exec -it sda-aws-container /bin/bash
+docker rm -f sda-aws-container >/dev/null 2>&1 || true