add: EXata Digital Twin component

exata_dt/.tnlcm/public.yaml

@@ -0,0 +1,53 @@
+######################################################
+##
+## Component public variables
+##
+## Diverse information read by the TNLCM
+## to adapt its behaviour depending on the component.
+##
+######################################################
+
+
+####################################
+## Component metadata
+####################################
+metadata:
+  maintainers:
+    - Matteo Pagin <matteo.pagin@keysight.com>
+    - Álvaro Curto Merino <alvaro.curtomerino@telefonica.com>
+  short_description: EXata Digital Twin appliance for KVM
+  long_description: |
+    Appliance featuring Keysight's EXata. The latter allows Experimenters to include EXata Digital Twins as part of a Trial Network, in turn enabling the programmatic retrieval of insights and predictions about the twinned live network.
+    This component requires a working EXata licensing server exposed to the Trial Networks in the site. The licensing server IP address can be provided in the 6G-Sandbox-Sites repository branch of the site.
+    The component has been implemented as a Docker Compose stack comprising two services: one hosting an instance of EXata, and the other hosting an instance of the official MariaDB service.
+    The interaction of Experimenters and/or the reminder of the Trial Network with the DT component takes place via northbound APIs exposed by a custom Python FastAPI webserver. The latter accepts HTTP requests and translates them into commands to the EXata and MariaDB Docker services.
+
+  hypervisors: ["one"]
+  appliances: ["https://marketplace.mobilesandbox.cloud:9443/appliance/EXata"]
+
+
+####################################
+## Site-specific variables
+####################################
+site_variables:
+  exata_licensing_server_ip: IP address of the EXata licensing server in your site.
+  template_id: ID of the Appliance to use in the OpenNebula environment
+  image_id: ID of the Appliance to use in the OpenNebula environment
+
+####################################
+## Input variables
+####################################
+input:
+  one_exata_dt_networks:
+    description: Ordered list of Virtual Network names the VM will be part of. The first Virtual Network is used by Jenkins to reach and configure the VM, so most cases will be fine using always the tn_vxlan
+    type: "list[tn_vxlan or vnet]"
+    default_value: ["tn_vxlan"]
+    required_when: false
+
+
+####################################
+## Generated terraform outputs
+####################################
+terraform_outputs:
+  __entity_name__id: "VM ID in OpenNebula. Generated from Terraform Manifest"
+  __entity_name__ips: "Dictionary of VM IP addresses: {<VNet ID in OpenNebula>: <IP address>}. Generated from Terraform Manifest"

exata_dt/README.md

@@ -0,0 +1,9 @@
+# exata_dt
+
+Deploy a VM with Keysight's EXata. The latter allows Experimenters to include EXata DTs as part of a Trial Network, in turn enabling the programmatic retrieval of insights and predictions about the twinned live network.
+
+This component requires a working EXata licensing server exposed to the Trial Networks in the site. The licensing server IP address can be provided in the 6G-Sandbox-Sites repository branch of the site.
+
+### Sample Trial Network using exata_dt
+
+[sample_tnlcm_descriptor_exata_dt](sample_tnlcm_descriptor_exata_dt.yaml)

exata_dt/changelog.md

@@ -0,0 +1,6 @@
+# Changelog
+
+## v0.6.0
+
+### Added
+- Initial release of `exata_dt` component to the 6G-Library.

exata_dt/code/component_playbook.yaml

@@ -0,0 +1,151 @@
+---
+- name: "STAGE 1: Apply IAC to deploy the component"
+  hosts: localhost
+  gather_facts: false
+  connection: local
+  tasks:
+    - name: Load enviromental variables from different sources
+      ansible.builtin.include_tasks: "{{ workspace }}/.global/cac/load_variables.yaml"
+
+    - name: Prepare terraform working directory
+      ansible.builtin.include_tasks: "{{ workspace }}/.global/cac/terraform_workdir.yaml"
+
+    - name: Configure selected VM resources
+      ansible.builtin.include_tasks: "{{ workspace }}/{{ component_type }}/code/{{ site_hypervisor }}/cac/01_pre/vm_resources.yaml"
+
+    - name: Terraform apply
+      ansible.builtin.include_tasks: "{{ workspace }}/.global/cac/terraform_apply.yaml"
+
+
+- name: "STAGE 2: Prepare to access the component"
+  hosts: localhost
+  gather_facts: false
+  connection: local
+  tasks:
+    - name: Retrieve terraform outputs
+      ansible.builtin.shell:
+      args:
+        chdir: "{{ workspace }}/.terraform/"
+        cmd: "set -o pipefail && terraform output --json | jq 'with_entries(.value |= .value)'"
+        executable: /bin/bash
+      register: terraform_outputs
+      changed_when: false
+
+    - name: Set Terraform outputs as playbook facts
+      ansible.builtin.set_fact:
+        bastion_ip: "{{ (terraform_outputs.stdout | from_json)['tn_bastion-ips'][site_networks_id.default | string] }}"
+        tn_ssh_public_key: "{{ (terraform_outputs.stdout | from_json)['tn_ssh_public_key'] }}"
+        tn_dns_token: "{{ (terraform_outputs.stdout | from_json)['tn_dns_token'] }}"
+        ips: "{{ (terraform_outputs.stdout | from_json)[entity_name + '-ips'] }}"
+        id: "{{ (terraform_outputs.stdout | from_json)[entity_name + '-id'] }}"
+        access_vnet_id: "{{ (terraform_outputs.stdout | from_json)[one_exata_dt_networks[0] + '-id'] | string }}"
+        domain_name: >-
+          {{
+            (
+              (component_type  | regex_replace('_', '') | lower) ~ '-' ~
+              (custom_name     | regex_replace('_', '') | lower) ~ '.' ~
+              (tn_id           | regex_replace('_', '') | lower) ~ '.' ~
+              site_domain
+            ) | trim
+          }}
+
+    - name: Add new VM to Ansible Inventory
+      ansible.builtin.add_host:
+        hostname: "exata_dt"
+        ansible_host: "{{ ips[access_vnet_id][0] }}"
+        ansible_ssh_common_args: "-J jenkins@{{ bastion_ip }}"
+        ansible_user: "jenkins"
+
+    - name: Add tn_bastion to Ansible Inventory
+      ansible.builtin.add_host:
+        hostname: "bastion"
+        ansible_host: "{{ bastion_ip }}"
+        ansible_user: "jenkins"
+
+    - name: Add new VM to SSH config file in the Jenkins-master for debugging purposes
+      ansible.builtin.include_tasks: "{{ workspace }}/exata_dt/code/{{ site_hypervisor }}/cac/01_pre/jenkins_ssh_config.yaml"
+
+    - name: Add new VM to an SSH config file ment to be usable by the experimenter
+      ansible.builtin.include_tasks: "{{ workspace }}/exata_dt/code/{{ site_hypervisor }}/cac/01_pre/tnuser_ssh_config.yaml"
+
+
+- name: "STAGE 3: Apply CAC to prepare the component"
+  hosts: "exata_dt"
+  gather_facts: false
+  tasks:
+    - name: Wait for system to become reachable
+      ansible.builtin.wait_for_connection:
+        connect_timeout: 5
+        timeout: 200
+
+    - name: Load enviromental variables from different sources
+      ansible.builtin.include_tasks: "{{ workspace }}/.global/cac/load_variables.yaml"
+
+    - name: Set site ssh key as authorized in jenkins user
+      ansible.posix.authorized_key:
+        user: jenkins
+        state: present
+        key: "{{ item }}"
+      loop:
+        - "{{ hostvars['localhost'].get('site_admin_ssh_public_key', '') }}"
+      when: item != ''
+
+    # User "tnuser" is already created as part of the appliance creation steps
+    - name: Set TN ssh key as authorized in new user
+      become: true
+      ansible.posix.authorized_key:
+        user: tnuser
+        state: present
+        key: "{{ item }}"
+      loop:
+        - "{{ hostvars['localhost'].get('tn_ssh_public_key', '') }}"
+
+    # The IP address of the EXata licensing server must be specified in the EXata license file.
+    # This step fetches the IP address of the EXata licensing server from the site variables, and dumps it
+    # into the host VM. Then, the latter is used by the component Docker entrypoint to overwrite the default
+    # EXata license file with the one that includes the IP address of the EXata licensing server.
+    - name: Write the EXata license file taking as input the IP addres of the EXata licensing server
+      ansible.builtin.shell: |
+        set -o pipefail
+        mkdir -p /home/EXata/webserver/exata-webserver-app/license
+        touch /home/EXata/webserver/exata-webserver-app/license/LicenseFile.lic
+        printf "SERVER {{ licensing_server_ip }} ANY\nVENDOR scalable\nUSE_SERVER" > /home/EXata/webserver/exata-webserver-app/license/LicenseFile.lic
+      args:
+        executable: /bin/bash
+      vars:
+        # Retrieve the EXata licensing server IP address for the platform the component is being deployed on
+        licensing_server_ip: "{{ hostvars.localhost.site_available_components.exata_dt.exata_licensing_server_ip | default('') }}"
+      become: true
+      changed_when: false
+
+# Whitelist the EXata licensing server in the bastion
+- name: "STAGE 4: Access tn_bastion to set firewall exceptions"
+  hosts: bastion
+  gather_facts: false
+  tasks:
+    - name: Write firewall exception for the EXata licensing server
+      ansible.builtin.include_tasks: "{{ workspace }}/.global/cac/nftables_add.yaml"
+      vars:
+        fw_exceptions:
+          - "{{ hostvars.localhost.site_available_components.exata_dt.exata_licensing_server_ip | default('') }}"
+
+
+- name: "STAGE 5: Publish execution results"
+  hosts: localhost
+  gather_facts: false
+  connection: local
+  tasks:
+    - name: Set DNS record for the component
+      ansible.builtin.uri:
+        url: >-
+          {{
+            'http://' ~ bastion_ip ~ ':5380/api/zones/records/add?token=' ~ tn_dns_token ~
+            '&domain=' ~ domain_name ~
+            '&type=A&ptr=true&createPtrZone=true&ipAddress=' ~ ips[access_vnet_id][0] | trim
+          }}
+        method: POST
+        body_format: json
+        status_code: [200]
+
+    - name: Publish execution results to TNLCM and S3 object storage
+      ansible.builtin.include_tasks: "{{ workspace }}/.global/cac/publish_ok_results.yaml"

exata_dt/code/one/cac/01_pre/jenkins_ssh_config.yaml

@@ -0,0 +1,10 @@
+---
+- name: Append new commponent's name and IP into a .ssh/config.d/ file
+  community.general.ssh_config:
+    host: "{{ tn_id }}-{{ entity_name }}"
+    hostname:  "{{ ips[access_vnet_id][0] | default('ERROR_ASSIGNING_IP', true) }}"
+    proxyjump: "{{ tn_id }}-tn_bastion"
+    remote_user: "jenkins"
+    ssh_config_file: "/var/lib/jenkins/.ssh/config.d/{{ tn_id }}"
+    state: present
+  register: jenkins_ssh_config

exata_dt/code/one/cac/01_pre/tnuser_ssh_config.yaml

@@ -0,0 +1,16 @@
+---
+### Module ssh_config has limited options and doesnt allow to specify identity files that doesn't exist
+- name: Append new component's name and IP into a .ssh/config.d/ file
+  ansible.builtin.blockinfile:
+    path: "{{ workspace }}/misc-{{ tn_id }}-ssh_config"
+    block: |
+      Host {{ tn_id }}-{{ entity_name }}
+          HostName "{{ ips[access_vnet_id][0] | default('ERROR_ASSIGNING_IP', true) }}"
+          User tnuser
+          IdentityFile ~/.ssh/misc-{{ tn_id }}-id_ed25519
+          StrictHostKeyChecking no
+          UserKnownHostsFile /dev/null
+          LogLevel QUIET
+    marker: "# {mark} {{ tn_id }}-{{ entity_name }} CONFIG"
+    create: true
+  register: tnuser_ssh_config

exata_dt/code/one/cac/01_pre/vm_resources.yaml

@@ -0,0 +1,5 @@
+---
+
+- name: Overwrite again with explicit values from the input file
+  ansible.builtin.include_vars:
+    file: "{{ workspace }}/{{ component_type }}/variables/input_file.yaml"

exata_dt/code/one/iac/exata_dt.tf.j2

@@ -0,0 +1,47 @@
+# https://registry.terraform.io/providers/OpenNebula/opennebula/latest/docs/resources/virtual_machine
+# https://docs.opennebula.io/6.8/management_and_operations/references/template.html#context-section
+resource "opennebula_virtual_machine" "{{ entity_name }}" {
+  name        = "{{ tn_id }}-{{ entity_name }}"
+  template_id = {{ site_available_components.exata_dt.template_id }}
+  cpu         = {{ one_exata_dt_cpu }}
+  vcpu        = {{ one_exata_dt_cpu }}
+  memory      = {{ one_exata_dt_memory }}
+
+  context = {
+    NETWORK =        "YES"
+    NETCFG_TYPE =    "interfaces"
+    SET_HOSTNAME =   "$NAME"
+    USERNAME =       "jenkins"
+    SSH_PUBLIC_KEY = "$USER[SSH_PUBLIC_KEY]"
+  }
+
+  # Ignore changes in the disk argument due to limitations in the OpenNebula Terraform provider (#598)
+  lifecycle {
+    ignore_changes = [disk]
+  }
+  disk {
+    image_id = {{ site_available_components.exata_dt.image_id }}
+    size     = {{ one_exata_dt_disk }}
+    target   = "vda"
+    driver   = "qcow2"
+  }
+
+{% for network in one_exata_dt_networks %}
+  nic {
+    model           = "virtio"
+    network_id      = opennebula_virtual_network.{{ network }}.id
+  }
+{% endfor %}
+
+  depends_on = [ opennebula_virtual_network_address_range.tn_vxlan ]
+}
+
+output "{{ entity_name }}-id" {
+  description = "OpenNebula VM ID"
+  value = opennebula_virtual_machine.{{ entity_name }}.id
+}
+
+output "{{ entity_name }}-ips" {
+  description = "OpenNebula VM Network Assignments"
+  value = { for nic in opennebula_virtual_machine.{{ entity_name }}.nic[*] : nic.network_id => nic.computed_ip... }
+}

exata_dt/result_templates/fail_result.md.j2

@@ -0,0 +1,5 @@
+There was an error. Detailed info:
+
+```
+{{ terraform_apply.stderr }}
+```

exata_dt/result_templates/ok_result.md.j2

@@ -0,0 +1,18 @@
+# {{ tn_id }}-{{ entity_name}}
+
+The component `{{ tn_id }}-{{ entity_name }}` has been succesfully created.
+
+It consists of an EXata VM, which allows Experimenters to include EXata DTs as part of a Trial Network, in turn enabling the programmatic retrieval of insights and predictions about the twinned live network.
+
+## Important information:
+
+- **OpenNebula VM ID**: `{{ id }}`
+- **VM memory**: `{{ one_exata_dt_memory}} MiB`
+- **VM VCPUs**: `{{ one_exata_dt_cpu }}`
+- **VM internal storage**: `{{ one_exata_dt_disk}} MiB`
+- **VM IP addresses**:
+{% for ip_list in ips.values() %}
+{% for ip in ip_list %}
+  - `{{ ip }}`
+{% endfor %}
+{% endfor %}

exata_dt/sample_input_file.yaml

@@ -0,0 +1,7 @@
+# THIS IS AN INPUT FILE EXAMPLE. Values may not be valid for your enviroment
+
+#This is a list of network ids. Each entry means a network interface attached to this network.
+one_exata_dt_networks: 
+  - "vnet_1"
+  - "vnet_2"
+  - "vnet_3"

exata_dt/sample_tnlcm_descriptor_exata_dt.yaml

@@ -0,0 +1,33 @@
+trial_network:
+  tn_init:
+    type: "tn_init"
+    dependencies: []
+    input: {}
+  vnet-n2:
+    type: "vnet"
+    name: "n2"
+    dependencies: 
+      - "tn_init"
+    input:
+      one_vnet_first_ip: "10.10.10.1"
+      one_vnet_netmask: 24
+      one_vnet_address_size: 254
+      # one_vnet_gw: null   # there is no gateway in this network
+      # one_vnet_dns: null  # there is no DNS in this network
+  # vnet-n3:
+  #   type: "vnet"
+  #   name: "n3"
+  #   dependencies:
+  #     - "tn_init"
+  #   input:
+  #     one_vnet_first_ip: "10.10.11.1"
+  #     one_vnet_netmask: 24
+  #     one_vnet_address_size: 254
+  #     # one_vnet_gw: null   # there is no gateway in this network
+  #     # one_vnet_dns: null  # there is no DNS in this network
+  exata_dt:
+    type: "exata_dt"
+    name: "exata_dt"
+    dependencies:
+      - "tn_init"
+      - "vnet-n2"

exata_dt/variables/one/private.yaml

@@ -0,0 +1,15 @@
+######################################################
+##
+## Component private variables
+##
+## Variables required for the deployment
+## and their default value.
+## Some of them are modifiable through input variables
+##
+######################################################
+
+one_exata_dt_cpu: 6
+one_exata_dt_memory: 8192
+one_exata_dt_disk: 42950
+one_exata_dt_networks: 
+  - tn_vxlan