Add CML Action to run whole workflow

.github/workflows/cml-report.yml

@@ -0,0 +1,122 @@
+name: CML Report
+on:
+  pull_request:
+    # types: [ labeled ]
+    branches:
+      - master
+
+jobs:
+  sim-summary:
+    # if:  ${{ github.event.label.name == ['run-simulation','GCBM'] }}
+    name: CML Simulation
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v3
+
+    - name: Setup python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+
+    # - name: Build API
+    #   run: |
+    #     python local/rest_api_gcbm/cml_report/process_input.py
+    #     cd local/rest_api_gcbm
+    #     docker build -f Dockerfile.cml --build-arg BUILD_TYPE=RELEASE --build-arg NUM_CPU=4 -t gcbm-api .
+    #     docker run -d --name gcbm-api --rm -p 8080:8080 gcbm-api
+
+    # - name: Get Public IP
+    #   id: ip
+    #   uses: haythem/public-ip@v1.2
+
+    # - name: Print public IP
+    #   run: |
+    #     echo ${{ steps.ip.outputs.ipv4 }}
+
+    # - name: Check ports
+    #   run: |
+    #     sudo netstat -tunlp
+
+    # - name: Create New Sim
+    #   run: |
+    #     curl -d "title=run4" -X POST http://0.0.0.0:8080/gcbm/new
+
+    # - name: Upload Files 
+    #   run: |
+    #     cd GCBM/GCBM_New_Demo_Run
+    #     curl -F disturbances='@disturbances/disturbances_2011_moja.tiff' \
+    #         -F disturbances='@disturbances/disturbances_2012_moja.tiff' \
+    #         -F disturbances='@disturbances/disturbances_2013_moja.tiff' \
+    #         -F disturbances='@disturbances/disturbances_2014_moja.tiff' \
+    #         -F disturbances='@disturbances/disturbances_2015_moja.tiff' \
+    #         -F disturbances='@disturbances/disturbances_2016_moja.tiff' \
+    #         -F disturbances='@disturbances/disturbances_2018_moja.tiff' \
+    #         -F classifiers='@classifiers/Classifier1_moja.tiff' \
+    #         -F classifiers='@classifiers/Classifier2_moja.tiff' \
+    #         -F db='@db/gcbm_input.db' \
+    #         -F miscellaneous='@miscellaneous/initial_age_moja.tiff' \
+    #         -F miscellaneous='@miscellaneous/mean_annual_temperature_moja.tiff' \
+    #         -F title="run4" \
+    #         http://localhost:8080/gcbm/upload
+
+    # - name: Run Simulation
+    #   run: |
+    #     pip install requests
+    #     python local/rest_api_gcbm/cml_report/run_gcbm.py
+
+    # - name: Retrive Logs and Download Results
+    #   run: |
+    #     cd local/tests
+    #     python simulation_status.py
+    #     cd ../..
+    #     curl -d "title=run4" -X POST http://0.0.0.0:8080/gcbm/download -L -o run4.zip
+    #     python local/rest_api_gcbm/cml_report/process_output.py
+
+    # - name: Upload Logs as artifact
+    #   uses: actions/upload-artifact@v3
+    #   with:
+    #     name: simulation-logs
+    #     path: local/tests/*.txt
+
+
+    - name: Build container
+      run: |
+        cd local/rest_api_gcbm
+        docker build -f Dockerfile.cml --build-arg BUILD_TYPE=RELEASE --build-arg NUM_CPU=4 -t gcbm-api .
+        docker run -d --name gcbm --rm -p 8080:8080 gcbm-api
+
+    - name: Process Input
+      run: |
+        docker exec -i gcbm python cml_report/process_input.py
+
+    - name: Run Simulation
+      run: |
+        docker exec -i -w /app/tests/config gcbm /opt/gcbm/moja.cli --config_file gcbm_config.cfg --config_provider provider_config.json
+        docker cp gcbm:/app/tests/output/logs/Moja_Debug.log local/rest_api_gcbm/tests
+
+    - name: Upload Logs as artifact
+      uses: actions/upload-artifact@v3
+      with:
+        name: simulation-logs
+        path: local/rest_api_gcbm/tests/Moja_Debug.log
+
+    - name: Compile Results
+      run: |
+        docker exec -i gcbm python GCBM.CompileResults/compileresults.py sqlite:///tests/output/gcbm_output.db --output_db sqlite:///tests/output/compiled_simulation_output.db
+        docker cp -L gcbm:/app/tests/output/compiled_simulation_output.db local/rest_api_gcbm/tests
+
+    - name: Setup CML
+      uses: iterative/setup-cml@v1
+
+    - name: Send the report
+      env:
+        repo_token: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        pip install matplotlib
+        pip install pandas
+        python local/rest_api_gcbm/cml_report/annual_summaries.py
+        echo "# GCBM Model" > report.md
+        echo "## Total Biomass Curve" >> report.md
+        cml-publish output/total_biomass_mt.png --md >> report.md
+        cml-send-comment report.md

.gitignore

@@ -173,3 +173,4 @@ terraform.rc
 # Add GCBM Demo runs
 GCBM_Demo_Run/
 GCBM_New_Demo_Run/
+local/rest_api_gcbm/cml_report/templates

local/rest_api_gcbm/Dockerfile

@@ -10,6 +10,10 @@ RUN pip install --no-cache-dir -r requirements.txt
 # Adding npm for CML Actions
 RUN apt-get update 
 RUN apt-get install nodejs npm -y
+RUN apt-get install -y python3-dev \
+    && cd /usr/local/bin \
+    && ln -s /usr/bin/python3 python
+RUN python --version
 RUN node --version
 RUN npm --version
 

local/rest_api_gcbm/Dockerfile.cml

@@ -0,0 +1,21 @@
+FROM ghcr.io/moja-global/flint.gcbm:master
+# FROM ghcr.io/moja-global/rest_api_gcbm@sha256:4c23b9874109b274247c26e3a01f8bdeecf26cda1afc8b5941234530c47c0012
+
+WORKDIR /app
+
+RUN pip install --upgrade pip
+
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+
+RUN apt-get update 
+RUN apt-get install -y python3-dev \
+    && cd /usr/local/bin \
+    && ln -s /usr/bin/python3 python
+RUN python --version
+
+RUN git clone https://github.com/moja-global/GCBM.CompileResults.git
+
+COPY . .
+
+CMD ["gunicorn", "--bind", ":8080", "--workers", "1", "--threads", "8", "--timeout", "0", "app:app"]

local/rest_api_gcbm/cml_report/annual_summaries.py

@@ -0,0 +1,19 @@
+import sqlite3
+import pandas as pd
+
+conn = sqlite3.connect("local/rest_api_gcbm/tests/output/compiled_simulation_output.db")
+
+query = f"""
+        SELECT years.year, COALESCE(SUM(i.pool_tc), 0) / 1e6 AS total_biomass_mt
+        FROM (SELECT DISTINCT year FROM v_age_indicators ORDER BY year) AS years
+        LEFT JOIN v_pool_indicators i
+            ON years.year = i.year
+        WHERE i.indicator = 'Total Biomass'
+            AND (years.year BETWEEN 2010 AND 2020)
+        GROUP BY years.year
+        ORDER BY years.year
+        """
+
+df = pd.read_sql_query(query, conn)
+ax = df.plot.line("year")  # x-axis
+ax.figure.savefig("local/rest_api_gcbm/tests/output/total_biomass_mt.png", dpi=300)

local/rest_api_gcbm/cml_report/process_input.py

@@ -0,0 +1,12 @@
+import zipfile
+import os
+
+#  Unzip file
+zip = os.path.join("tests","linux-demo.zip")
+with zipfile.ZipFile(zip, "r") as zip_ref:
+    zip_ref.extractall("tests")
+
+# template = os.path.join("local", "rest_api_gcbm", "templates.zip")
+# extract_to = os.path.join("local", "rest_api_gcbm", "cml_report")
+# with zipfile.ZipFile(template, "r") as zip_ref:
+#     zip_ref.extractall(extract_to)

local/rest_api_gcbm/cml_report/process_output.py

@@ -0,0 +1,6 @@
+import os
+import zipfile
+
+zip = "run4.zip"
+with zipfile.ZipFile(zip, "r") as zip_ref:
+    zip_ref.extractall("output")

local/rest_api_gcbm/cml_report/run_gcbm.py

@@ -0,0 +1,52 @@
+import os
+import requests
+from threading import Timer
+import time
+import json
+
+HOME = "http://localhost:8080/gcbm"
+DONE = False
+
+
+def run_gcbm():
+    url = f"{HOME}/dynamic"
+    req_data = {"title": "run4"}
+    res = requests.post(url, data=req_data)
+    return res.text
+
+
+def status():
+    url = f"{HOME}/status"
+    req_data = {"title": "run4"}
+    res = requests.post(url, data=req_data)
+    return json.loads(res.text)
+
+
+def display():
+    global DONE
+    sim_status = status()
+    if sim_status["finished"] == "In Progress":
+        print(os.system("docker logs gcbm-api --until=30s"))
+    else:
+        DONE = True
+
+
+class CheckSimulation(Timer):
+    def run(self):
+        while not self.finished.wait(self.interval):
+            self.function(*self.args, **self.kwargs)
+        print("Threading Done")
+
+
+run_sim = run_gcbm()
+print(run_sim)
+
+timer = CheckSimulation(30, display)
+print("Starting simulation.")
+timer.start()
+time.sleep(300)
+if not DONE:
+    time.sleep(300)
+
+print("Simulation Ending.")
+timer.cancel()