Update install.py to handle external django apps with server mode

.github/workflows/webui_rh_test.yml → .github/workflows/devel_server_install.yaml

@@ -1,5 +1,5 @@
 ---
-name: Test WebUI on RH based OS
+name: Devel Server Installation of OQ on RH OS
 on:
   workflow_dispatch:
     inputs:
@@ -12,16 +12,15 @@ on:
 
 jobs:
   test_ui:
-    name: Test on RH based (fedora container)
+    name: Test on RH based (fedora container) ${{ matrix.python-version }}
     runs-on: ubuntu-latest
     env:
       GITHUB_DEF_BR:  ${{ github.event.repository.default_branch }}
     strategy:
       matrix:
-        python-version: [python3.12]
-        req-version: [py312]
+        python-version: ["python3.11", "python3.12", "python3.13"]
     container:
-      image: fedora:41
+      image: fedora:44
       options: --privileged
     steps:
     - name: Clone Repository (Master)
@@ -41,15 +40,12 @@ jobs:
         else
            BRANCH=${{ github.event.inputs.git-ref }}
         fi
-        # Add openquake user
-        useradd -m -u 1000 -s /bin/bash openquake
-        # update system and install utilities
+        # Update system and install utilities
         dnf update -y
         dnf install -y ${{ matrix.python-version }}
-        dnf install -y git git-lfs nc procps
+        dnf install -y git git-lfs nc procps sudo
         # define it
         mypython=${{ matrix.python-version }}
-        reqpython=${{ matrix.req-version }}
         #run it
         eval '$mypython -c "import sys; print(sys.version)"'
         #
@@ -59,40 +55,10 @@ jobs:
         #
         echo "branch to test: ${BRANCH}"
         cd /__w/oq-engine/oq-engine/
-        eval '$mypython -m pip install -U pip setuptools wheel'
-        eval '$mypython -m venv /opt/openquake/venv '
-        source /opt/openquake/venv/bin/activate
-        eval '$mypython -m pip install -r "requirements-$reqpython-linux64.txt"'
-        eval '$mypython -m pip install -e . '
-        ## Standalone apps
-        echo "Downloading standalone apps"
-        for app in oq-platform-standalone django-gem-taxonomy oq-platform-ipt oq-platform-taxonomy; do
-            for branch in "$BRANCH" "master" "main"; do
-                TOOLS_BRANCH="$branch"
-                # do not fail on exit code 2 if the branch
-                # does not exist in the app repository
-                set +e
-                git ls-remote --exit-code --heads https://github.com/gem/${app}.git $TOOLS_BRANCH >/dev/null 2>&1
-                EXIT_CODE=$?
-                # set again to fail on exit code not 0
-                set -e
-                if [[ $EXIT_CODE -eq 0 ]]; then
-                    echo "Git branch '$BRANCH' exists in the remote repository"
-                    break
-                fi
-            done
-            if [[ $EXIT_CODE -ne 0 ]]; then
+        eval '$mypython install.py --version $BRANCH devel_server'
+        if [[ $EXIT_CODE -ne 0 ]]; then
                 break
-            fi
-            echo "We need to use the branch $TOOLS_BRANCH for the standalone apps"
-            git clone -b ${TOOLS_BRANCH} --depth=1 https://github.com/gem/${app}.git
-            eval '$mypython -m pip install -e ./${app}'
-        done
-        if [ $EXIT_CODE -ne 0 ]; then
-            echo "No '$branch', nor 'master' or 'main' branch found for '$app' django app; failed"
-            exit 1
         fi
-        deactivate
     - name: Actualize 'default' templates for email notifications
       run: |
         cd /__w/oq-engine/oq-engine/
@@ -103,7 +69,6 @@ jobs:
       run: |
          set -o pipefail
          source /opt/openquake/venv/bin/activate
-         pip freeze
          mkdir -p /var/log/oq-engine/
          chown -R openquake /var/log/oq-engine/
          runuser -l openquake -c '/opt/openquake/venv/bin/oq engine --upgrade-db'

.github/workflows/server_install.yaml

@@ -0,0 +1,180 @@
+---
+name: Server Installation of OQ
+on:
+  workflow_dispatch:
+    inputs:
+      git-ref:
+        description: Git Ref Branch
+        default: master
+        required: true
+      debug_enabled:
+        type: boolean
+        description: "Run the build with tmate debugging enabled"
+        required: false
+        default: false
+  schedule:
+    - cron: "05 11 * * *"
+
+jobs:
+  Sever_Installation:
+    name: Server OQ  Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_DEF_BR: ${{ github.event.repository.default_branch }}
+      GITHUB_REF: ${{ github.ref }}
+      GITHUB_HD_REF: ${{ github.head_ref }}}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - name: Clone Repository (Master)
+        uses: actions/checkout@v6
+        if: github.event.inputs.git-ref == ''
+      - name: Clone Repository (Custom Ref)
+        uses: actions/checkout@v6
+        if: github.event.inputs.git-ref != ''
+        with:
+          ref: ${{ github.event.inputs.git-ref }}
+      - name: Set up Python  ${{ matrix.python-version }}
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install OQ Server with ${{ matrix.python-version }}
+        run: |
+          if [ -z ${{ github.event.inputs.git-ref }} ];
+          then
+             BRANCH=master
+          else
+             BRANCH=${{ github.event.inputs.git-ref }}
+          fi
+          sudo -H "$(which python3)" install.py --version $BRANCH server
+          if [[ $EXIT_CODE -ne 0 ]]; then
+                break
+          fi
+      - name: Setup debugging session
+        uses: mxschmitt/action-tmate@v3
+        timeout-minutes: 60
+        if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
+      - name: Actualize 'default' templates for email notifications
+        run: |
+          SITE_PKG="/opt/openquake/venv/lib/python${{ matrix.python-version }}/site-packages/"
+          cd ${SITE_PKG}
+          for file in openquake/server/templates/registration/*.default.tmpl; do
+             echo ' cp -- "$file" "${file%.default.tmpl}" '
+             cp -- "$file" "${file%.default.tmpl}"
+          done
+          ls -lrt ${SITE_PKG}/openquake/server/templates/registration/
+      - name: WebUI and calculation with Auth
+        shell: sudo su root -c "bash --noprofile --norc -e -o pipefail {0}"
+        run: |
+          whoami  # Outputs: root
+          source /opt/openquake/venv/bin/activate
+          mkdir -p /var/log/oq-engine/
+          chown -R openquake /var/log/oq-engine/
+          runuser -l openquake -c '/opt/openquake/venv/bin/oq engine --upgrade-db'
+          oq --version
+          #
+          echo "Add settings for login and logging on webui before to start"
+          SITE_PKG="/opt/openquake/venv/lib/python${{ matrix.python-version }}/site-packages/"
+          cd  ${SITE_PKG}
+          cd openquake/server
+          cat > local_settings.py << EOF
+          APPLICATION_MODE = "RESTRICTED"
+          EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
+          EMAIL_HOST = 'smtp.gmail.com'
+          EMAIL_PORT = 587
+          EMAIL_USE_TLS = True
+          EMAIL_HOST_USER = 'admin@reply.com'
+          EMAIL_HOST_PASSWORD = 'polhjuih@'
+          EMAIL_SUPPORT = 'noreply@reply.com'
+          WEBUI_ACCESS_LOG_DIR = '/var/log/oq-engine'
+          EOF
+          echo "DISPLAY content of local_settings"
+          cat local_settings.py
+          # Perform migration after setup local_settings
+          /opt/openquake/venv/bin/python3 manage.py migrate
+          /opt/openquake/venv/bin/python3 manage.py createuser openquake testuser1@test.com --level 1 --password level_1_password --no-email
+          # STOP AND START WEBUI
+          systemctl stop openquake-webui
+          sleep 10
+          systemctl start openquake-webui
+          echo "Waiting WEBUI up on port 8800...."
+          echo "Test WebUI with curl before to test django"
+          while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' http://127.0.0.1:8800/accounts/login/)" != "200" ]]; do
+            echo "Still waiting WEBUI account login up on port 8800...."
+            sleep 5 # wait for 5 seconds before check again
+          done
+          echo -n "TEST DJANGO LOGIN "
+          LOGIN_URL=http://127.0.0.1:8800/accounts/login/
+          YOUR_USER='openquake'
+          YOUR_PASS='level_1_password'
+          COOKIES=cookies.txt
+          CURL_BIN="curl -s -c $COOKIES -b $COOKIES -e $LOGIN_URL"
+          echo -n "Django Auth: get csrftoken ..."
+          $CURL_BIN $LOGIN_URL -o /dev/null
+          DJANGO_TOKEN="csrfmiddlewaretoken=$(grep csrftoken $COOKIES | sed 's/^.*csrftoken\s*//')"
+          echo " perform login ..."
+          status_code=$($CURL_BIN -w "%{http_code}"\
+             -d "$DJANGO_TOKEN&username=$YOUR_USER&password=$YOUR_PASS" \
+             -X POST $LOGIN_URL -o /dev/null)
+          if [ $status_code != "302" ]; then
+           echo "Unable to login"
+           exit 1
+          fi
+          echo "--------------------"
+          echo "display log of webui"
+          echo "--------------------"
+          cat /var/log/oq-engine/webui-access.log
+          if [ -s /var/log/oq-engine/webui-access.log ]; then
+           # The file is not-empty.
+           cat /var/log/oq-engine/webui-access.log
+          else
+           echo "/var/log/oq-engine/webui-access.log is empty. Something did not work as expected"
+           exit 1
+          fi
+          echo " Run a calc using https"
+          runuser -l openquake -c '/opt/openquake/venv/bin/oq engine --run https://downloads.openquake.org/jobs/M4_Exercise.zip'
+          sleep 10
+          status_code=$($CURL_BIN -w "%{http_code}"\
+             -d "$DJANGO_TOKEN&username=$YOUR_USER&password=$YOUR_PASS" \
+             -G http://127.0.0.1:8800/engine/1/outputs -o /dev/null)
+          if [ $status_code != "200" ]; then
+           echo "Unable to visualize outputs"
+           exit 1
+          fi
+      - name: WebUI on PUBLIC
+        shell: sudo su root -c "bash --noprofile --norc -e -o pipefail {0}"
+        run: |
+          set -o pipefail
+          source /opt/openquake/venv/bin/activate
+          echo "Restart WebUI without authentication"
+          systemctl stop openquake-webui
+          sleep 5
+          runuser -l openquake -c 'OQ_APPLICATION_MODE=PUBLIC /opt/openquake/venv/bin/oq webui start 127.0.0.1:8800 -s &'
+          echo "Waiting WEBUI up on port 8800...."
+          while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' http://127.0.0.1:8800/engine/)" != "200" ]]; do
+           echo "Still waiting WEBUI on port 8800...."
+           sleep 5 # wait for 5 seconds before check again
+          done
+          echo -n "Test Standalone Tools pages (without authentication)"
+          read status_code redirect_url <<<"$(
+           curl -s -o /dev/null -w "%{http_code} %{redirect_url}" http://127.0.0.1:8800/
+          )"
+          if [ "$status_code" != "301" ] || [ "$redirect_url" != "http://127.0.0.1:8800/engine/" ]; then
+           echo "Unexpected redirect: status=$status_code redirect=$redirect_url"
+           echo "Should redirect to http://127.0.0.1:8800/engine/"
+           exit 1
+          fi
+          status_code=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8800/taxonomy/)
+          if [ $status_code != "200" ]; then
+           echo "Unable access the taxonomy standalone app"
+           exit 1
+          fi
+          status_code=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8800/ipt/)
+          if [ $status_code != "200" ]; then
+           echo "Unable to access the ipt standalone app"
+           exit 1
+          fi
+          echo " Run a calc using https"
+          runuser -l openquake -c '/opt/openquake/venv/bin/oq engine --run https://downloads.openquake.org/jobs/eb_risk_demo.zip'