diff --git a/local/rest_api_gcbm/README.md b/local/rest_api_gcbm/README.md
index fae9e057..a464ddea 100644
--- a/local/rest_api_gcbm/README.md
+++ b/local/rest_api_gcbm/README.md
@@ -1,38 +1,43 @@
-# FLINT.Cloud
-
-### GCBM local-run REST API Setup
-
-Run the GCBM local-run container by pushing the following command:
+This script follows the pseudocode from Andrew's design to update GCBM configs: https://hackmd.io/@aornugent/B1RluG69c.
+
+## Execution
+
+### End-Point
+
+* Endpoints supported: `/gcbm/upload`
+* Categories supported: `disturbances, classifiers, miscellaneous`
```bash
-docker-compose up
+# This uploads a classifier, disturbance and a miscellaneous file
+curl -F disturbances="@disturbances/disturbances_2011_moja.tiff" -F classifiers="@classifiers/Classifier1_moja.tiff" http://localhost:8080/gcbm/upload
+
+# Try skipping disturbance file, and it will still work :)
```
-Navigate to http://localhost:8080/ in the browser. You can stop the running container by pushing the following command:
+**Expected Output:**
```bash
-docker-compose down
+{
+ "error": "Missing files for categories: ['config_files', 'input'], they are required for the simulation to run"
+}
```
-Currently the REST API has the following endpoints available for access:-
-
-| Endpoint | Functionality | Method |
-| :----------------: | :----------------: | :----------------: |
-| **\help\all** | This endpoint produces a help message with information on all options for moja.CLI | `GET`
-| **\help\arg** | This endpoint produces a help message with information on option arg for moja.CLI. | `GET`
-| **\gcbm\new** | This endpoint creates a new directory to store input and output files of the simulation. Parameter to be passed in the body is the title of the new simulation or default value simulation will be used. | `POST` |
-| **\gcbm\upload** | This endpoint is used to upload the input files: config_files, input and database. Remember to upload the files in the body from the GCBM folder which is available in the root of this repository for setup and use. A directory /input will be created to store the specified files. | `POST` |
-| **\gcbm\dynamic** | This endpoint runs simulation in a thread and generates an output zip file in /output directory. The process may take a while. Parameter is the title of the simulation. | `POST` |
-| **\gcbm\status** | This endpoint is employed to check the status of the simulation. It sends a message 'Output is ready to download' to notify that the output zip file is generated. Parameter is the title of the simulation. | `POST`
-| **\gcbm\download** | This endpoint is used to download the output zip file. Parameter is the title of the simulation. | `POST`
-| **\gcbm\list** | This endpoint retrieves the complete list of simulations that are created using /new. | `GET`
+The outputs for classifier file and disturbance file will be stored in "templates/config/" folder.
-The inputs are contained in `GCBM_Demo_Run.zip`, present in the root of the directory. This file must be unzipped for further usage.
+### GCBM Pre-Processing
- Once the container is up and running, the following methods can be used to interact with the endpoints
+Run: `python3 gcbm.py disturbances/disturbances_2011_moja.tiff` and a folder will be generated with the output config file (JSON)
+
+Script:
+
+```bash
+pip install -r requirements.txt
+python3 gcbm.py disturbances/disturbances_2011_moja.tiff
+cat templates/config/disturbances_2011_moja.json
+```
-1. A sample Postman collection is available [here](https://github.com/nynaalekhya/FLINT.Cloud/blob/local-gcbm-run/rest_local_run/local_run.postman_collection). Import the collection to Postman and run the endpoints.
+Please see the `main` function in the `gcbm.py` file for more info. A payload is manually added, which checks if it has year and then the `has_year` key is set to `True`
-2. The endpoints can be interacted with using `cURL`.
-curl is used in command lines or scripts to transfer data. Find out more about curl [here](https://curl.se/). Commands using cURL can be found [here](curl.md)
+## TODOs
+There are a lot of in-line TODOs, this is the first draft to see how we can implement this the most pythonic way.
\ No newline at end of file
diff --git a/local/rest_api_gcbm/app.py b/local/rest_api_gcbm/app.py
index 6021c916..cbed1544 100644
--- a/local/rest_api_gcbm/app.py
+++ b/local/rest_api_gcbm/app.py
@@ -1,769 +1,58 @@
-from threading import Thread
-
-from numpy import append
-from run_distributed import *
-from flask_autoindex import AutoIndex
-from flask_swagger_ui import get_swaggerui_blueprint
-from flask_swagger import swagger
-from datetime import timedelta
-from datetime import datetime
-from google.cloud import storage, pubsub_v1
-from google.api_core.exceptions import AlreadyExists
-import logging
-import shutil
import json
-import time
-import subprocess
import os
-import flask.scaffold
-import rasterio as rst
-from flask import jsonify
-from config_table import rename_columns
-import sqlite3
-
-flask.helpers._endpoint_from_view_func = flask.scaffold._endpoint_from_view_func
-from flask_restful import Resource, Api, reqparse
-from flask import Flask, send_from_directory, request, jsonify, redirect, send_file
+from flask import Flask
+from flask import request
+from flask_restful import Api
from flask_cors import CORS
+from enum import Enum
+
+from gcbm import GCBMSimulation
app = Flask(__name__)
-CORS(
- app,
- origins=[
- "http://127.0.0.1:8080/",
- "http://127.0.0.1:8000",
- "http://localhost:5000",
- "http://localhost:8000",
- r"^https://.+example.com$",
- ],
-)
-# ppath = "/"
-# AutoIndex(app, browse_root=ppath)
+CORS(app, origins=["http://127.0.0.1:8080/"])
api = Api(app)
-# logger config
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
-c_handler = logging.StreamHandler()
-c_handler.setLevel(logging.DEBUG)
-c_format = logging.Formatter("%(name)s - %(levelname)s - %(message)s")
-c_handler.setFormatter(c_format)
-logger.addHandler(c_handler)
-
-
-### swagger specific ###
-SWAGGER_URL = "/swagger"
-API_URL = "/static/swagger.json"
-SWAGGERUI_BLUEPRINT = get_swaggerui_blueprint(
- SWAGGER_URL, API_URL, config={"app_name": "FLINT-GCBM REST API"}
-)
-app.register_blueprint(SWAGGERUI_BLUEPRINT, url_prefix=SWAGGER_URL)
-### end swagger specific ###
-
-
-@app.route("/spec")
-def spec():
- swag = swagger(app)
- swag["info"]["version"] = "1.0"
- swag["info"]["title"] = "FLINT-GCBM Rest Api"
- f = open("./static/swagger.json", "w+")
- json.dump(swag, f)
- return jsonify(swag)
-
-
-@app.route("/help/", methods=["GET"])
-def help(arg):
- """
- Get Help Section
- ---
- tags:
- - help
- parameters:
- - name: arg
- in: path
- description: Help info about named section. Pass all to get all info
- required: true
- type: string
- responses:
- 200:
- description: Help
- """
- s = time.time()
- if arg == "all":
- res = subprocess.run(["/opt/gcbm/moja.cli", "--help"], stdout=subprocess.PIPE)
- else:
- res = subprocess.run(
- ["/opt/gcbm/moja.cli", "--help-section", arg], stdout=subprocess.PIPE
- )
- e = time.time()
-
- response = {
- "exitCode": res.returncode,
- "execTime": e - s,
- "response": res.stdout.decode("utf-8"),
- }
- return {"data": response}, 200
-
-
-@app.route("/version", methods=["GET"])
-def version():
- """
- Get Version of FLINT
- ---
- tags:
- - version
- responses:
- 200:
- description: Version
- """
- s = time.time()
- res = subprocess.run(["/opt/gcbm/moja.cli", "--version"], stdout=subprocess.PIPE)
- e = time.time()
- response = {
- "exitCode": res.returncode,
- "execTime": e - s,
- "response": res.stdout.decode("utf-8"),
- }
- return {"data": response}, 200
+class REQUIRED(Enum):
+ YES = 1
+ NO = 0
-@app.route("/gcbm/new", methods=["POST"])
-def gcbm_new():
- """
- Create a new GCBM simulation with a title.
- ---
- tags:
- - gcbm
- responses:
- 200:
- parameters:
- - in: body
- name: title
- required: true
- schema:
- type: string
- description: Create a new simulation for GCBM Implementation of FLINT
- """
- # Default title = simulation
- title = request.form.get("title") or "simulation"
- # Sanitize title
- title = "".join(c for c in title if c.isalnum())
- # input_dir = f"{title}"
- input_dir = f"{os.getcwd()}/input/{title}"
- if not os.path.exists(f"{input_dir}"):
- os.makedirs(f"{input_dir}")
- message = "New simulation started. Please move on to the next stage for uploading files at /gcbm/upload."
- else:
- message = "Simulation already exists. Please check the list of simulations present before proceeding with a new simulation at gcbm/list. You may also download the input and output files for this simulation at gcbm/download sending parameter title in the body."
-
- return {"data": message}, 200
+categories = {
+ "disturbances": REQUIRED.NO,
+ "classifiers": REQUIRED.YES,
+ "miscellaneous": REQUIRED.YES,
+ "config_files": REQUIRED.YES,
+ "input": REQUIRED.YES
+}
@app.route("/gcbm/upload", methods=["POST"])
def gcbm_upload():
- """
- Upload files for GCBM Dynamic implementation of FLINT
- ---
- tags:
- - gcbm
- responses:
- 200:
- parameters:
- - in: body
- name: title
- required: true
- schema:
- type: string
- description: File upload for GCBM Implementation FLINT
- """
-
- # Default title = simulation
- title = request.form.get("title") or "simulation"
- # Sanitize title
- title = "".join(c for c in title if c.isalnum())
-
- # Create project directory
- input_dir = f"{os.getcwd()}/input/{title}"
- if not os.path.exists(f"{input_dir}"):
- os.makedirs(f"{input_dir}")
- logging.debug(os.getcwd())
-
- # input files follow a strict structure
- if not os.path.exists(f"{input_dir}/disturbances"):
- os.makedirs(f"{input_dir}/disturbances")
- if not os.path.exists(f"{input_dir}/classifiers"):
- os.makedirs(f"{input_dir}/classifiers")
- if not os.path.exists(f"{input_dir}/db"):
- os.makedirs(f"{input_dir}/db")
- if not os.path.exists(f"{input_dir}/miscellaneous"):
- os.makedirs(f"{input_dir}/miscellaneous")
-
- # store files following structure defined in curl.md
- if "disturbances" in request.files:
- for file in request.files.getlist("disturbances"):
- file.save(f"{input_dir}/disturbances/{file.filename}")
- else:
- return {"error": "Missing configuration file"}, 400
-
- if "classifiers" in request.files:
- for file in request.files.getlist("classifiers"):
- file.save(f"{input_dir}/classifiers/{file.filename}")
- else:
- return {"error": "Missing configuration file"}, 400
-
- if "db" in request.files:
- for file in request.files.getlist("db"):
- file.save(f"{input_dir}/db/{file.filename}")
- else:
- return {"error": "Missing configuration file"}, 400
-
- if "miscellaneous" in request.files:
- for file in request.files.getlist("miscellaneous"):
- file.save(f"{input_dir}/miscellaneous/{file.filename}")
- else:
- return {"error": "Missing configuration file"}, 400
-
- get_config_templates(input_dir)
- get_modules_cbm_config(input_dir)
- get_provider_config(input_dir)
-
- return {
- "data": "All files uploaded succesfully. Proceed to the next step of the API at gcbm/dynamic."
- }
-
-
-def get_config_templates(input_dir):
- if not os.path.exists(f"{input_dir}/templates"):
- shutil.copytree(
- f"{os.getcwd()}/templates", f"{input_dir}/templates", dirs_exist_ok=False
- )
-
-
-# TODO: there needs to be a link between the files configured here append
-# the ["vars"] attribute of modules_cbm.json -> CBMDisturbanceListener
-# current hack is to drop the last five characters, but thats very fragile
-def get_modules_cbm_config(input_dir):
- with open(f"{input_dir}/templates/modules_cbm.json", "r+") as modules_cbm_config:
- disturbances = []
- data = json.load(modules_cbm_config)
- for file in os.listdir(f"{input_dir}/disturbances/"):
- disturbances.append(
- file.split(".")[0][:-5]
- ) # drop `_moja` to match modules_cbm.json template
- modules_cbm_config.seek(0)
- data["Modules"]["CBMDisturbanceListener"]["settings"]["vars"] = disturbances
- json.dump(data, modules_cbm_config, indent=4)
- modules_cbm_config.truncate()
-
-
-def get_provider_config(input_dir):
- with open(f"{input_dir}/templates/provider_config.json", "r+") as provider_config:
- lst = []
- data = json.load(provider_config)
-
- for file in os.listdir(f"{input_dir}/db/"):
- d = dict()
- d["path"] = file
- d["type"] = "SQLite"
- data["Providers"]["SQLite"] = d
- provider_config.seek(0)
-
- for file in os.listdir(f"{input_dir}/disturbances/"):
- d = dict()
- d["name"] = file[:-10]
- d["layer_path"] = file
- d["layer_prefix"] = file[:-5]
- lst.append(d)
- provider_config.seek(0)
- data["Providers"]["RasterTiled"]["layers"] = lst
-
- for file in os.listdir(f"{input_dir}/classifiers/"):
- d = dict()
- d["name"] = file[:-10]
- d["layer_path"] = file
- d["layer_prefix"] = file[:-5]
- lst.append(d)
- provider_config.seek(0)
- data["Providers"]["RasterTiled"]["layers"] = lst
-
- for file in os.listdir(f"{input_dir}/miscellaneous/"):
- d = dict()
- d["name"] = file[:-10]
- d["layer_path"] = file
- d["layer_prefix"] = file[:-5]
- lst.append(d)
- provider_config.seek(0)
- data["Providers"]["RasterTiled"]["layers"] = lst
-
- Rasters = []
- Rastersm = []
- nodatam = []
- nodata = []
- cellLatSize = []
- cellLonSize = []
- paths = []
-
- for root, dirs, files in os.walk(os.path.abspath(f"{input_dir}/disturbances/")):
- for file in files:
- fp = os.path.join(root, file)
- Rasters.append(fp)
- paths.append(fp)
-
- for root, dirs, files in os.walk(os.path.abspath(f"{input_dir}/classifiers/")):
- for file in files:
- fp1 = os.path.join(root, file)
- Rasters.append(fp1)
- paths.append(fp1)
-
- for nd in Rasters:
- img = rst.open(nd)
- t = img.transform
- x = t[0]
- y = -t[4]
- n = img.nodata
- cellLatSize.append(x)
- cellLonSize.append(y)
- nodata.append(n)
-
- result = all(element == cellLatSize[0] for element in cellLatSize)
- if result:
- cellLat = x
- cellLon = y
- nd = n
- blockLat = x * 400
- blockLon = y * 400
- tileLat = x * 4000
- tileLon = y * 4000
- else:
- print("Corrupt files")
-
- provider_config.seek(0)
-
- data["Providers"]["RasterTiled"]["cellLonSize"] = cellLon
- data["Providers"]["RasterTiled"]["cellLatSize"] = cellLat
- data["Providers"]["RasterTiled"]["blockLonSize"] = blockLon
- data["Providers"]["RasterTiled"]["blockLatSize"] = blockLat
- data["Providers"]["RasterTiled"]["tileLatSize"] = tileLat
- data["Providers"]["RasterTiled"]["tileLonSize"] = tileLon
-
- json.dump(data, provider_config, indent=4)
- provider_config.truncate()
-
- dictionary = {
- "layer_type": "GridLayer",
- "layer_data": "Byte",
- "nodata": nd,
- "tileLatSize": tileLat,
- "tileLonSize": tileLon,
- "blockLatSize": blockLat,
- "blockLonSize": blockLon,
- "cellLatSize": cellLat,
- "cellLonSize": cellLon,
- }
-
- # should be able to accept variable number of inputs, but requires
- # means for user to specify/verify correct ["attributes"]
- def get_input_layers():
- for root, dirs, files in os.walk(
- os.path.abspath(f"{input_dir}/miscellaneous/")
- ):
- for file in files:
- fp2 = os.path.join(root, file)
- Rastersm.append(fp2)
-
- for i in Rastersm:
- img = rst.open(i)
- d = img.nodata
- nodatam.append(d)
-
- with open(
- f"{input_dir}/initial_age_moja.json", "w", encoding="utf8"
- ) as json_file:
- dictionary["layer_type"] = "GridLayer"
- dictionary["layer_data"] = "Int16"
- dictionary["nodata"] = nodatam[1]
- json.dump(dictionary, json_file, indent=4)
-
- with open(
- f"{input_dir}/mean_annual_temperature_moja.json", "w", encoding="utf8"
- ) as json_file:
- dictionary["layer_type"] = "GridLayer"
- dictionary["layer_data"] = "Float32"
- dictionary["nodata"] = nodatam[0]
- json.dump(dictionary, json_file, indent=4)
-
- with open(
- f"{input_dir}/Classifier1_moja.json", "w", encoding="utf8"
- ) as json_file:
- dictionary["layer_type"] = "GridLayer"
- dictionary["layer_data"] = "Byte"
- dictionary["nodata"] = nd
- dictionary["attributes"] = {
- "1": "TA",
- "2": "BP",
- "3": "BS",
- "4": "JP",
- "5": "WS",
- "6": "WB",
- "7": "BF",
- "8": "GA",
- }
- json.dump(dictionary, json_file, indent=4)
-
- with open(
- f"{input_dir}/Classifier2_moja.json", "w", encoding="utf8"
- ) as json_file:
- dictionary["layer_type"] = "GridLayer"
- dictionary["layer_data"] = "Byte"
- dictionary["nodata"] = nd
- dictionary["attributes"] = {"1": "5", "2": "6", "3": "7", "4": "8"}
- json.dump(dictionary, json_file, indent=4)
-
- with open(
- f"{input_dir}/disturbances_2011_moja.json", "w", encoding="utf8"
- ) as json_file:
- dictionary["layer_data"] = "Byte"
- dictionary["nodata"] = nd
- dictionary["attributes"] = {
- "1": {"year": 2011, "disturbance_type": "Wildfire", "transition": 1}
- }
- json.dump(dictionary, json_file, indent=4)
-
- with open(
- f"{input_dir}/disturbances_2012_moja.json", "w", encoding="utf8"
- ) as json_file:
- dictionary["attributes"] = {
- "1": {"year": 2012, "disturbance_type": "Wildfire", "transition": 1}
- }
- json.dump(dictionary, json_file, indent=4)
-
- with open(
- f"{input_dir}/disturbances_2013_moja.json", "w", encoding="utf8"
- ) as json_file:
- dictionary["attributes"] = {
- "1": {
- "year": 2013,
- "disturbance_type": "Mountain pine beetle — Very severe impact",
- "transition": 1,
- },
- "2": {
- "year": 2013,
- "disturbance_type": "Wildfire",
- "transition": 1,
- },
- }
- json.dump(dictionary, json_file, indent=4)
-
- with open(
- f"{input_dir}/disturbances_2014_moja.json", "w", encoding="utf8"
- ) as json_file:
- dictionary["attributes"] = {
- "1": {
- "year": 2014,
- "disturbance_type": "Mountain pine beetle — Very severe impact",
- "transition": 1,
- }
- }
- json.dump(dictionary, json_file, indent=4)
-
- with open(
- f"{input_dir}/disturbances_2015_moja.json", "w", encoding="utf8"
- ) as json_file:
- dictionary["attributes"] = {
- "1": {"year": 2016, "disturbance_type": "Wildfire", "transition": 1}
- }
- json.dump(dictionary, json_file, indent=4)
-
- with open(
- f"{input_dir}/disturbances_2016_moja.json", "w", encoding="utf8"
- ) as json_file:
- dictionary["attributes"] = {
- "1": {"year": 2016, "disturbance_type": "Wildfire", "transition": 1}
- }
- json.dump(dictionary, json_file, indent=4)
-
- with open(
- f"{input_dir}/disturbances_2018_moja.json", "w", encoding="utf8"
- ) as json_file:
- dictionary["attributes"] = {
- "1": {
- "year": 2018,
- "disturbance_type": "Mountain pine beetle — Low impact",
- "transition": 1,
- }
- }
- json.dump(dictionary, json_file, indent=4)
-
- get_input_layers()
-
- def get_study_area():
- study_area = {
- "tile_size": tileLat,
- "block_size": blockLat,
- "tiles": [
- {
- "x": int(t[2]),
- "y": int(t[5]),
- "index": 12674,
- }
- ],
- "pixel_size": cellLat,
- "layers": [],
- }
-
- with open(f"{input_dir}/study_area.json", "w", encoding="utf") as json_file:
- list = []
-
- for file in os.listdir(f"{input_dir}/miscellaneous/"):
- d1 = dict()
- d1["name"] = file[:-10]
- d1["type"] = "VectorLayer"
- list.append(d1)
- study_area["layers"] = list
-
- for file in os.listdir(f"{input_dir}/classifiers/"):
- d1 = dict()
- d1["name"] = file[:-10]
- d1["type"] = "VectorLayer"
- d1["tags"] = ["classifier"]
- list.append(d1)
- study_area["layers"] = list
-
- for file in os.listdir(f"{input_dir}/disturbances/"):
- d1 = dict()
- d1["name"] = file[:-10]
- d1["type"] = "DisturbanceLayer"
- d1["tags"] = ["disturbance"]
- list.append(d1)
- study_area["layers"] = list
-
- json.dump(study_area, json_file, indent=4)
-
- get_study_area()
-
- for root, dirs, files in os.walk(os.path.abspath(f"{input_dir}/disturbances/")):
- for file in files:
- fp = os.path.join(root, file)
- Rasters.append(fp)
- paths.append(fp)
-
- for root, dirs, files in os.walk(os.path.abspath(f"{input_dir}/classifiers/")):
- for file in files:
- fp1 = os.path.join(root, file)
- Rasters.append(fp1)
- paths.append(fp1)
-
- for root, dirs, files in os.walk(
- os.path.abspath(f"{input_dir}/miscellaneous/")
- ):
- for file in files:
- fp2 = os.path.join(root, file)
- paths.append(fp2)
-
- for root, dirs, files in os.walk(os.path.abspath(f"{input_dir}/templates/")):
- for file in files:
- fp3 = os.path.join(root, file)
- paths.append(fp3)
-
- for root, dirs, files in os.walk(os.path.abspath(f"{input_dir}/db/")):
- for file in files:
- fp4 = os.path.join(root, file)
- paths.append(fp4)
-
- # copy files to input directory
- for i in paths:
- print(i)
- shutil.copy2(i, (f"{input_dir}"))
-
- # delete folders from input directory
- shutil.rmtree((f"{input_dir}/disturbances/"))
- shutil.rmtree((f"{input_dir}/templates/"))
- shutil.rmtree((f"{input_dir}/classifiers/"))
- shutil.rmtree((f"{input_dir}/miscellaneous/"))
- shutil.rmtree((f"{input_dir}/db/"))
-
-
-@app.route("/config", methods=["POST"])
-def config_table():
- obj = request.get_json()
- print(obj)
- input_dir = f"{os.getcwd()}/input/{obj['simulation_name']}"
- response = dict()
- try:
- return {
- "status": 1,
- "response": rename_columns(obj["tables"], obj["simulation_name"]),
- }
- except Exception:
- return {"status": 0, "error": Exception}
-
-
-@app.route("/gcbm/dynamic", methods=["POST"])
-def gcbm_dynamic():
- """
- Get GCBM Dynamic implementation of FLINT
- ---
- tags:
- - gcbm
- responses:
- 200:
- parameters:
- - in: body
- name: title
- required: true
- schema:
- type: string
- description: GCBM Implementation FLINT
- """
- # Default title = simulation
- title = request.form.get("title") or "simulation"
-
- # Sanitize title
- title = "".join(c for c in title if c.isalnum())
- input_dir = f"{os.getcwd()}/input/{title}"
-
- gcbm_config_path = "gcbm_config.cfg"
- provider_config_path = "provider_config.json"
-
- if not os.path.exists(f"{input_dir}"):
- os.makedirs(f"{input_dir}")
-
- thread = Thread(target=launch_run, kwargs={"title": title, "input_dir": input_dir})
- thread.start()
- # subscriber_path = create_topic_and_sub(title)
- return {"status": "Run started"}, 200
-
-
-def launch_run(title, input_dir):
- s = time.time()
- logging.debug("Starting run")
- with open(f"{input_dir}/gcbm_logs.csv", "w+") as f:
- res = subprocess.Popen(
- [
- "/opt/gcbm/moja.cli",
- "--config_file",
- "gcbm_config.cfg",
- "--config_provider",
- "provider_config.json",
- ],
- stdout=f,
- cwd=f"{input_dir}",
- )
- logging.debug("Communicating")
- (output, err) = res.communicate()
- logging.debug("Communicated")
-
- if not os.path.exists(f"{input_dir}/output"):
- logging.error(err)
- return "OK"
- logging.debug("Output exists")
-
- # cut and paste output folder to app/output/simulation_name
- shutil.copytree(f"{input_dir}/output", (f"{os.getcwd()}/output/{title}"))
- shutil.make_archive(
- f"{os.getcwd()}/output/{title}", "zip", f"{os.getcwd()}/output/{title}"
- )
- shutil.rmtree((f"{input_dir}/output"))
- logging.debug("Made archive")
- e = time.time()
-
- logging.debug("Generated URL")
- response = {
- "exitCode": res.returncode,
- "execTime": e - s,
- "response": "Operation executed successfully. Downloadable links for input and output are attached in the response. Alternatively, you may also download this simulation input and output results by making a request at gcbm/download with the title in the body.",
- }
-
-
-@app.route("/gcbm/download", methods=["POST"])
-def gcbm_download():
- """
- Download GCBM Input and Output
- ---
- tags:
- - gcbm
- responses:
- 200:
- parameters:
- - in: body
- name: title
- required: true
- schema:
- type: string
- description: GCBM Download FLINT
- """
- # Default title = simulation
- title = request.form.get("title") or "simulation"
- # Sanitize title
- title = "".join(c for c in title if c.isalnum())
- return send_file(
- f"{os.getcwd()}/output/{title}.zip",
- attachment_filename="{title}.zip",
- )
-
-
-@app.route("/gcbm/list", methods=["GET"])
-def gcbm_list_simulations():
- """
- Get GCBM Simulations List
- ---
- tags:
- - gcbm
- responses:
- 200:
- description: GCBM Simulations List
- """
-
- list = []
- for file in os.listdir(f"{os.getcwd()}/input"):
- list.append(file)
-
- return {
- "data": list,
- "message": "To create a new simulation, create a request at gcbm/new. To access the results of the existing simulations, create a request at gcbm/download.",
- }, 200
-
-
-@app.route("/gcbm/status", methods=["POST"])
-def status():
- """
- Get status of a simulation
- ---
- tags:
- - gcbm
- responses:
- 200:
- parameters:
- - in: body
- name: title
- required: true
- schema:
- type: string
- description: Get status of simulation
- """
- # Default title = simulation
- title = request.form.get("title") or "simulation"
- # Sanitize title
- title = "".join(c for c in title if c.isalnum())
-
- if os.path.isfile(f"{os.getcwd()}/input/{title}/output.zip"):
- message = "Output is ready to download at gcbm/download"
- else:
- message = "In Progress"
+ title = request.form.get('title') or 'simulation'
+ title = ''.join(c for c in title if c.isalnum())
- return {"finished": message}
+ project_dir = f"{title}"
+ if not os.path.exists(f"{os.getcwd()}/input/{project_dir}"):
+ os.makedirs(f"{os.getcwd()}/input/{project_dir}")
+ def fix_path(path):
+ return os.path.basename(path.replace("\\", "/"))
-@app.route("/check", methods=["GET", "POST"])
-def check():
- return "Checks OK", 200
+ gcbm = GCBMSimulation()
+ errored_categories = []
+ for category, req in categories.items():
+ if category in request.files:
+ for file in request.files.getlist(category):
+ gcbm.add_file(category + "/" + file.filename)
+ elif req == REQUIRED.YES:
+ errored_categories.append(category)
-@app.route("/", methods=["GET"])
-def home():
- return "FLINT.Cloud API"
+ if len(errored_categories) != 0:
+ return {"error": f"Missing files for categories: {errored_categories}, they are required for the simulation to run"}, 400
+ return {"data": "All files uploaded successfully."}, 200
if __name__ == "__main__":
- app.run(debug=True, host="0.0.0.0", port=int(os.environ.get("PORT", 8080)))
+ app.run(debug=True, host='0.0.0.0', port=int(os.environ.get("PORT", 8080)))
\ No newline at end of file
diff --git a/local/rest_api_gcbm/gcbm.py b/local/rest_api_gcbm/gcbm.py
new file mode 100644
index 00000000..665acc15
--- /dev/null
+++ b/local/rest_api_gcbm/gcbm.py
@@ -0,0 +1,301 @@
+import os
+import pathlib
+import json
+
+import rasterio
+
+
+class GCBMList:
+ """
+ This is a base class for GCBM pre-processing scripts to use. It prevents users to do: ._append()
+ """
+
+ def __init__(self, files=dict(), config=[], category=None):
+ self.data = list()
+ self.files = files
+ self.config = config
+ self.category = category
+
+ def __iter__(self):
+ return self.data
+
+ def __getitem__(self, idx):
+ return self.data[idx]
+
+ def is_category(self, path):
+ if self.category is None:
+ raise NotImplementedError(
+ "Please implement `is_category` method, which is used by _append() method"
+ )
+ else:
+ return self.category in path
+
+ # Unlike list.append() in Python, this returns a bool - whether the append was successful or not + checks if the file path is of the current category
+ def _append(self, file_path):
+ if self.is_category(file_path):
+ self.data.append(file_path)
+ return True
+ return False
+
+ def _update_config(self):
+ for raster_file_path in self.data:
+ json_config_file_path = GCBMList.change_extension(raster_file_path, ".json")
+
+ if json_config_file_path.name not in self.config:
+ self.generate_config(
+ os.path.abspath(raster_file_path), json_config_file_path
+ )
+ else:
+ with open(
+ f"templates/config/{json_config_file_path.name}", "r+"
+ ) as _file:
+ json.dump(
+ self.files[os.path.abspath(raster_file_path)], _file, indent=4
+ )
+
+ def _populate_config_with_hard_coded_config(
+ self, config, hc_config, nodata
+ ):
+ # Note: hc_config => hard_coded_config
+ for key in hc_config.keys():
+ if key.startswith("_"):
+ # the format is: __: (index is useless here, TODO: remove it)
+ original_key = key.split("_")[1]
+ if hc_config[key] is None:
+ continue
+ else:
+ config[original_key] = nodata
+ else:
+ config[key] = hc_config[key]
+ return config
+
+ def generate_config(self, file_path, json_config_file_path):
+ mode = "w+"
+ if os.path.exists(f"templates/config/{json_config_file_path}"):
+ mode = "r+"
+
+ hard_coded_path = f"hard_coded_values/{json_config_file_path}"
+ hard_coded_config = None
+ if os.path.exists(hard_coded_path):
+ with open(hard_coded_path) as hard_coded_file:
+ try:
+ hard_coded_config = json.load(hard_coded_file)
+ except json.decoder.JSONDecodeError as e:
+ raise e
+
+ with open(f"templates/config/{json_config_file_path.name}", mode) as _file:
+ if mode == "r+":
+ config = json.load(_file)
+ else:
+ config = dict()
+
+ # Defaults
+ with rasterio.open(file_path) as raster_obj:
+ tr = raster_obj.transform
+ config["cellLatSize"] = tr[0]
+ config["cellLonSize"] = -tr[4]
+ config["nodata"] = raster_obj.nodata
+
+ config["blockLonSize"] = config["cellLonSize"] * 400
+ config["blockLatSize"] = config["cellLatSize"] * 400
+ config["tileLatSize"] = config["cellLatSize"] * 4000
+ config["tileLonSize"] = config["cellLonSize"] * 4000
+ config["layer_type"] = "GridLayer"
+ config["layer_data"] = "Byte"
+ # config["has_year"] = False
+ # config["has_type"] = False
+
+ # Now populate if hard_coded_config exists
+ config = self._populate_config_with_hard_coded_config(
+ config, hard_coded_config, raster_obj.nodata
+ )
+
+ print("Dumping config: ", config)
+ json.dump(config, _file, indent=4)
+
+ self.files[file_path] = config
+ self.config.append(json_config_file_path.name)
+
+ # TODO:
+ # self.sync_config()
+
+ def setattr(self, file, attributes):
+ file = os.path.abspath(file)
+ config = self.files[file]
+ config["attributes"] = attributes
+
+ if config["attributes"]["year"]:
+ config["has_year"] = True
+
+ self.files[file] = config
+ self._update_config()
+
+ @staticmethod
+ def change_extension(file_path, new_extension):
+ # TODO: let's use pathlib.Path everywhere, for now it's okay here
+ pathlib_path = pathlib.Path(file_path)
+ return pathlib_path.with_suffix(new_extension)
+
+
+class GCBMDisturbanceList(GCBMList):
+ def __init__(self, files, config):
+ category = "disturbances"
+ self.files = files
+ self.config = config
+ super().__init__(files=files, config=config, category=category)
+
+
+class GCBMClassifiersList(GCBMList):
+ def __init__(self, files, config):
+ category = "classifiers"
+ self.files = files
+ self.config = config
+ super().__init__(category=category)
+
+
+class GCBMMiscellaneousList(GCBMList):
+ def __init__(self, files, config):
+ category = "miscellaneous"
+ self.files = files
+ self.config = config
+ super().__init__(category=category)
+
+
+class GCBMSimulation:
+ def __init__(self):
+ # create a global index
+ self.files = {}
+
+ # create sub-indices of different types
+ self.config = list()
+ self.parameters = [] # this is the input_db
+
+ self.create_simulation_folder()
+ self.create_file_index()
+
+ self.classifiers = GCBMClassifiersList(files=self.files, config=self.config)
+ self.disturbances = GCBMDisturbanceList(files=self.files, config=self.config)
+ self.miscellaneous = GCBMMiscellaneousList(files=self.files, config=self.config)
+
+ def create_simulation_folder(self):
+ if not os.path.exists("templates/config"):
+ os.makedirs("templates/config/")
+
+ def create_file_index(self):
+ config_dir_path = "templates/config"
+ assert os.path.isdir(
+ config_dir_path
+ ), f"Given config directory path: {config_dir_path} either does not exist or is not a directory."
+ for dirpath, _, filenames in os.walk(config_dir_path):
+ for filename in filenames:
+ # Don't read any data, but create the json file
+ abs_filepath = os.path.abspath(os.path.join(dirpath, filename))
+
+ data = GCBMSimulation.safe_read_json(abs_filepath)
+
+ # TODO: Discussion - should this be abs_filepath, or do we want just the filename?
+ self.files[abs_filepath] = data
+
+ # TODO: This should not happen here? maybe connect an endpoint directly to the sync_config method
+ # self.sync_config(abs_filepath)
+
+ # file_path: disturbances (NOT MUST), classifiers (MUST), miscellaneous (MUST)
+ def add_file(self, file_path: str):
+ """
+ This function:
+
+ 1. Checks if the given file is one of the categories: registers, classifiers, and miscellaneous.
+ 2. The provided file path to the buffer, and updates the config (JSON).
+
+ Parameters
+ ==========
+ 1. file_path (str), no default
+ """
+ if self.disturbances._append(file_path):
+ self.disturbances._update_config()
+ return
+ if self.classifiers._append(file_path):
+ self.classifiers._update_config()
+ return
+ if self.miscellaneous._append(file_path):
+ self.miscellaneous._update_config()
+ return
+ # TODO: Add covariates here
+
+ # TODO
+ # self._save(file_path)
+
+ def sync_config(self, file_path):
+ def _write_to_file(file_path, data):
+ with open(file_path, "w+") as _file:
+ _file.write(data)
+
+ data = GCBMSimulation.safe_read_json(file_path)
+
+ if self.files[file_path] != data:
+ # Means data has changed, so update the file_path
+ _write_to_file(file_path, data)
+ # Also update the dict
+ self.files[file_path] = data
+
+ # TODO (@ankitaS11): We can just have these as class methods later, this will reduce the redundancy in the code later
+ def update_disturbance_config(self):
+ self.disturbances._update_config()
+
+ def set_disturbance_attributes(self, file, payload):
+ self.disturbances.setattr(file, payload)
+
+ def update_classifier_config(self):
+ self.classifiers._update_config()
+
+ def set_classifier_attributes(self, file, payload):
+ self.classifiers.setattr(file, payload)
+
+ def update_miscellaneous_config(self):
+ self.miscellaneous._update_config()
+
+ def set_miscellaneous_attributes(self, file, payload):
+ self.miscellaneous.setattr(file, payload)
+
+ @staticmethod
+ def safe_read_json(path):
+ if ".json" not in path:
+ raise UserWarning(f"Given path {path} not a json file")
+ return {}
+ # Make sure it's a file and not a directory
+ if not os.path.isfile(path):
+ raise UserWarning(
+ f"Got a directory {path} inside the config directory path, skipping it."
+ )
+ return {}
+ with open(path, "r") as json_file:
+ data = json.load(json_file)
+ return data
+
+
+if __name__ == "__main__":
+ sim = GCBMSimulation()
+ sim.add_file("disturbances/disturbances_2011_moja.tiff")
+ sim.add_file("disturbances/disturbances_2012_moja.tiff")
+ sim.add_file("disturbances/disturbances_2013_moja.tiff")
+ sim.add_file("disturbances/disturbances_2014_moja.tiff")
+ sim.add_file("disturbances/disturbances_2015_moja.tiff")
+ sim.add_file("disturbances/disturbances_2016_moja.tiff")
+ sim.add_file("disturbances/disturbances_2018_moja.tiff")
+ sim.add_file("classifiers/Classifier1_moja.tiff")
+ sim.add_file("classifiers/Classifier2_moja.tiff")
+ sim.add_file("miscellaneous/initial_age_moja.tiff")
+ sim.add_file("miscellaneous/mean_annual_temperature_moja.tiff")
+ # this^ generates disturbances_2011_moja.json file, which will contain the metadata from .tiff
+
+ # Sample payload to test
+ # If you want to add something of your own to the json file above, you can do this:
+ # payload = {
+ # "year": 2012,
+ # "disturbance_type": "Wildfire",
+ # "random_thing": [1, 2, 3, 4],
+ # "transition": 1,
+ # }
+ # sim.set_disturbance_attributes("disturbances/disturbances_2011_moja.tiff", payload)
+ # sim.set_classifier_attributes("classifiers/classifier1_moja.tiff", payload)
+ # sim.set_miscellaneous_attributes("miscellaneous/initial_age_moja.tiff", payload)
diff --git a/local/rest_api_gcbm/requirements.txt b/local/rest_api_gcbm/requirements.txt
index 270092fb..3129820a 100644
--- a/local/rest_api_gcbm/requirements.txt
+++ b/local/rest_api_gcbm/requirements.txt
@@ -1,15 +1,4 @@
-flask==2.0.3
-flask-restful==0.3.9
-gunicorn==20.1.0
-flask-swagger==0.2.14
-flask_swagger_ui==3.36.0
-flask_autoindex==0.6.6
-google-cloud-storage==2.2.1
-google-cloud-pubsub==2.11.0
-flask-cors==3.0.10
-simplejson==3.17.6
-sqlalchemy==1.4.32
-psutil==5.9.0
-pandas==1.4.2
-matplotlib==3.5.2
-rasterio
\ No newline at end of file
+rasterio
+flask
+flask_cors
+flask_restful
\ No newline at end of file