diff --git a/.github/workflows/run-onecc-build.yml b/.github/workflows/run-onecc-build.yml index 0c31395b9ac..e21d94dd973 100644 --- a/.github/workflows/run-onecc-build.yml +++ b/.github/workflows/run-onecc-build.yml @@ -45,12 +45,14 @@ jobs: strategy: matrix: type: [ Debug, Release ] - ubuntu_code: [ focal, jammy ] + ubuntu_code: [ focal, jammy ] #, noble ] include: - ubuntu_code: focal ubuntu_ver: 20.04 - ubuntu_code: jammy ubuntu_ver: 22.04 + #- ubuntu_code: noble + # ubuntu_ver: 24.04 runs-on: one-x64-linux container: image: samsungonedev.azurecr.io/nnfw/one-devtools:${{ matrix.ubuntu_code }} @@ -67,9 +69,15 @@ jobs: # dalgona uses pybind11, but pybind11 cannot bind packages in virtualenv. # So we need to install packages for dalgona-test globally. - name: Install required packages + if: ${{ matrix.ubuntu_code == 'focal' || matrix.ubuntu_code == 'jammy' }} run: | python3 -m pip install numpy h5py==3.11.0 flatbuffers==23.5.26 + - name: Install required packages + if: ${{ matrix.ubuntu_code == 'noble' }} + run: | + apt-get install -y python3-numpy python3-h5py python3-flatbuffers + - name: Caching externals uses: actions/cache@v4 with: diff --git a/.github/workflows/run-onert-android-build.yml b/.github/workflows/run-onert-android-build.yml index 0e3146f976a..c238c595497 100644 --- a/.github/workflows/run-onert-android-build.yml +++ b/.github/workflows/run-onert-android-build.yml @@ -21,13 +21,6 @@ on: - master - release/* paths: - - '.github/workflows/run-onert-android-build.yml' - - 'nn*' - - 'Makefile.template' - - 'infra/buildtool/**' - - 'infra/cmake/**' - - 'infra/nncc/**' - - 'infra/nnfw/**' - 'runtime/**' - '!runtime/contrib/**' - '!**/*.md' diff --git a/.github/workflows/run-onert-cross-build.yml b/.github/workflows/run-onert-cross-build.yml index 76d8e8fd0f5..1f86d858d4b 100644 --- a/.github/workflows/run-onert-cross-build.yml +++ b/.github/workflows/run-onert-cross-build.yml @@ -21,13 +21,6 @@ on: - master - release/* paths: - - '.github/workflows/run-onert-cross-build.yml' - - 'nn*' - - 'Makefile.template' - - 'infra/buildtool/**' - - 'infra/cmake/**' - - 'infra/nncc/**' - - 'infra/nnfw/**' - 'runtime/**' - '!runtime/contrib/**' - '!**/*.md' diff --git a/.github/workflows/run-onert-gbs-build.yml b/.github/workflows/run-onert-gbs-build.yml index f95f0a8c0fc..b614de2368f 100644 --- a/.github/workflows/run-onert-gbs-build.yml +++ b/.github/workflows/run-onert-gbs-build.yml @@ -24,15 +24,6 @@ on: - master - release/* paths: - - '.github/workflows/run-onert-gbs-build.yml' - - 'nncc' - - 'nnfw' - - 'compiler/luci/**' - - 'compiler/loco/**' - - 'compiler/logo/**' - - 'compute/**' - - 'infra/nnfw/**' - - 'infra/cmake/**' - 'runtime/**' - '!runtime/contrib/**' - 'packaging/**' diff --git a/.github/workflows/run-onert-native-build.yml b/.github/workflows/run-onert-native-build.yml index 6374cb32ad7..b001a2e0e72 100644 --- a/.github/workflows/run-onert-native-build.yml +++ b/.github/workflows/run-onert-native-build.yml @@ -21,13 +21,6 @@ on: - master - release/* paths: - - '.github/workflows/run-onert-native-build.yml' - - 'nn*' - - 'Makefile.template' - - 'infra/buildtool/**' - - 'infra/cmake/**' - - 'infra/nncc/**' - - 'infra/nnfw/**' - 'runtime/**' - '!runtime/contrib/**' - '!**/*.md' diff --git a/compiler/CMakeLists.txt b/compiler/CMakeLists.txt index 580bd21f41e..4ea8e16c622 100644 --- a/compiler/CMakeLists.txt +++ b/compiler/CMakeLists.txt @@ -3,7 +3,7 @@ set(ONE_UBUNTU_CODENAME "focal") find_program(LSB_RELEASE_EXEC lsb_release) if(LSB_RELEASE_EXEC) - # output should be one of 'bionic', 'focal', 'jammy' + # output should be one of 'bionic', 'focal', 'jammy', 'noble' # others are not tested execute_process(COMMAND "${LSB_RELEASE_EXEC}" --short --codename OUTPUT_VARIABLE ONE_UBUNTU_CODENAME @@ -14,6 +14,10 @@ endif() if(${ONE_UBUNTU_CODENAME} STREQUAL "bionic") set(ONE_UBUNTU_CODENAME_BIONIC TRUE) +elseif(${ONE_UBUNTU_CODENAME} STREQUAL "focal") + set(ONE_UBUNTU_CODENAME_FOCAL TRUE) +elseif(${ONE_UBUNTU_CODENAME} STREQUAL "jammy") + set(ONE_UBUNTU_CODENAME_JAMMY TRUE) elseif(${ONE_UBUNTU_CODENAME} STREQUAL "noble") set(ONE_UBUNTU_CODENAME_NOBLE TRUE) endif() diff --git a/compiler/one-cmds/CMakeLists.txt b/compiler/one-cmds/CMakeLists.txt index 2e43fce4faa..f0dafc83729 100644 --- a/compiler/one-cmds/CMakeLists.txt +++ b/compiler/one-cmds/CMakeLists.txt @@ -2,7 +2,7 @@ # Ubuntu18.04; explictly installed python3.8 (default is python3.6) # Ubuntu20.04; default python3.8 # Ubuntu22.04; default python3.10 -# Ubuntu24.04; explicitly installed python3.8 (default is python3.12) +# Ubuntu24.04; default python3.12 # refer https://github.com/Samsung/ONE/issues/9962 if(CMAKE_VERSION VERSION_LESS 3.12) find_package(PythonInterp 3.8 QUIET) @@ -24,10 +24,10 @@ else() endif() # tensorflow 2.12.1 supports Python 3.8 ~ 3.11 - if(Python_VERSION VERSION_GREATER_EQUAL 3.12) - message(STATUS "Build one-cmds: FALSE (Python version 3.12 or higher is not supported yet)") - return() - endif() + #if(Python_VERSION VERSION_GREATER_EQUAL 3.12) + # message(STATUS "Build one-cmds: FALSE (Python version 3.12 or higher is not supported yet)") + # return() + #endif() if(Python_VERSION VERSION_LESS 3.8) message(STATUS "Build one-cmds: FAILED (Install Python version 3.8 or 3.10)") return() @@ -67,17 +67,7 @@ if(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "aarch64") # and install build/../one-prepare-venv file list(APPEND ONE_COMMAND_FILES one-prepare-venv.aarch64) else(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "aarch64") - if(ONE_UBUNTU_CODENAME_BIONIC) - # NOTE copy one-prepare-venv.u1804 as build/../one-prepare-venv - # and install build/../one-prepare-venv file - list(APPEND ONE_COMMAND_FILES one-prepare-venv.u1804) - elseif(ONE_UBUNTU_CODENAME_NOBLE) - # NOTE copy one-prepare-venv.u2404 as build/../one-prepare-venv - # and install build/../one-prepare-venv file - list(APPEND ONE_COMMAND_FILES one-prepare-venv.u2404) - else() - list(APPEND ONE_COMMAND_FILES one-prepare-venv) - endif() + list(APPEND ONE_COMMAND_FILES one-prepare-venv) endif(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "aarch64") # pytorch importer is an experimental feature, it is not used in default configuration diff --git a/compiler/one-cmds/one-import-onnx b/compiler/one-cmds/one-import-onnx index 4a1960e332c..e9212ab9325 100644 --- a/compiler/one-cmds/one-import-onnx +++ b/compiler/one-cmds/one-import-onnx @@ -24,7 +24,13 @@ import os import sys import tempfile import onnx -import onnx_tf + +# onnx_tf may not exist for latest python +try: + import onnx_tf + _onnx_tf_enabled = True +except ImportError: + _onnx_tf_enabled = False # ONNX legalizer is an optional feature # It enables conversion of some operations, but in experimental phase for now @@ -124,6 +130,10 @@ class TidyIONames: self.onnx_model.graph.output[i].name = self.remap_outputs[idx] +def _check_onnx_tf(): + import onnx + + def get_driver_cfg_section(): return "one-import-onnx" @@ -330,6 +340,9 @@ def _convert(args): if _force_ext(args): run_default_import = False + if not _onnx_tf_enabled: + run_default_import = False + res_conv = -1 if run_default_import: if _force_ext(args): diff --git a/compiler/one-cmds/one-prepare-venv b/compiler/one-cmds/one-prepare-venv index 1803ec2da43..02e4e0e3d04 100644 --- a/compiler/one-cmds/one-prepare-venv +++ b/compiler/one-cmds/one-prepare-venv @@ -54,6 +54,7 @@ source /etc/lsb-release # - https://github.com/onnx/onnx/blob/master/docs/Versioning.md # - https://github.com/onnx/onnx-tensorflow/blob/master/Versioning.md +# default version values VER_TENSORFLOW=2.12.1 VER_TF_PROB=0.20.1 VER_TF_ADON=0.20.0 @@ -63,18 +64,31 @@ VER_ONNX_TF=1.10.0 VER_PYDOT=1.4.2 VER_TORCH="2.1.2+cpu" VER_PROTOBUF=4.23.3 - -# Install tensorflow +VER_NUMPY=1.24.3 echo "Setting version for '$DISTRIB_CODENAME'" if [[ "$DISTRIB_CODENAME" == "bionic" ]]; then - : # use as is + : # use default elif [[ "$DISTRIB_CODENAME" == "focal" ]]; then - : # use as is + : # use default elif [[ "$DISTRIB_CODENAME" == "jammy" ]]; then - : # TODO change vesions + : + #VER_TENSORFLOW=2.19.0 + #VER_TF_PROB=0.25.0 + #VER_TF_ADON=0.23.0 + #VER_ONNX=1.17.0 + #VER_ONNXRUNTIME=1.21.1 + #VER_TORCH="2.7.0+cpu" + #VER_NUMPY=1.24.4 elif [[ "$DISTRIB_CODENAME" == "noble" ]]; then - : # TODO change vesions + VER_TENSORFLOW=2.19.0 + VER_TF_PROB=0 + VER_TF_ADON=0 # EOL, don't exist in python3.12~ + VER_ONNX=1.17.0 + VER_ONNXRUNTIME=1.21.0 + VER_ONNX_TF=0 + VER_TORCH="2.7.0+cpu" + VER_NUMPY=1.26.4 else echo "Error one-prepare-venv: Unsupported codename $DISTRIB_CODENAME" exit -1 @@ -97,15 +111,18 @@ if [[ ! -z "$ONE_PREPVENV_PIP_OPTION" ]]; then fi ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install --upgrade pip setuptools -if [ -n "${EXT_TENSORFLOW_WHL}" ]; then - ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install ${EXT_TENSORFLOW_WHL} -else - ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install tensorflow-cpu==${VER_TENSORFLOW} + +PYTHON_PACKAGES="tensorflow-cpu==${VER_TENSORFLOW} " + +PYTHON_PACKAGES+="Pillow " + +if [[ "$VER_TF_PROB" != "0" ]]; then + # Fix version to that of TF release date + PYTHON_PACKAGES+="tensorflow_probability==${VER_TF_PROB} " +fi +if [[ "$VER_TF_ADON" != "0" ]]; then + PYTHON_PACKAGES+="tensorflow_addons==${VER_TF_ADON} " fi -${VENV_PYTHON} -m pip ${PIP_OPTIONS} install Pillow -# Fix version to that of TF release date -${VENV_PYTHON} -m pip ${PIP_OPTIONS} install tensorflow_probability==${VER_TF_PROB} -${VENV_PYTHON} -m pip ${PIP_OPTIONS} install tensorflow_addons==${VER_TF_ADON} # Install PyTorch and ONNX related # NOTE set ONE_PREPVENV_TORCH_SOURCE to override options for source URL. @@ -115,24 +132,26 @@ if [[ ! -z "$ONE_PREPVENV_TORCH_SOURCE" ]]; then fi # TODO remove torch message echo "Torch from '${ONE_PREPVENV_TORCH_SOURCE}' -> '${TORCH_SOURCE_OPTION}'" -${VENV_PYTHON} -m pip ${PIP_OPTIONS} install torch==${VER_TORCH} ${TORCH_SOURCE_OPTION} +PYTHON_PACKAGES+="torch==${VER_TORCH} " -${VENV_PYTHON} -m pip ${PIP_OPTIONS} install onnx==${VER_ONNX} +PYTHON_PACKAGES+="onnx==${VER_ONNX} " -${VENV_PYTHON} -m pip ${PIP_OPTIONS} install onnxruntime==${VER_ONNXRUNTIME} +PYTHON_PACKAGES+="onnxruntime==${VER_ONNXRUNTIME} " -# Provide install of custom onnx-tf -if [ -n "${EXT_ONNX_TF_WHL}" ]; then - ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install ${EXT_ONNX_TF_WHL} -else - ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install onnx-tf==${VER_ONNX_TF} +if [[ "$VER_ONNX_TF" != "0" ]]; then + PYTHON_PACKAGES+="onnx-tf==${VER_ONNX_TF} " fi # Fix version to that of TF release date -${VENV_PYTHON} -m pip ${PIP_OPTIONS} install --upgrade protobuf==${VER_PROTOBUF} +PYTHON_PACKAGES+="protobuf==${VER_PROTOBUF} " -# Fix version with fsspec error on U20.04 -${VENV_PYTHON} -m pip ${PIP_OPTIONS} install --upgrade fsspec==2024.6.1 +# Fix version with fsspec error +PYTHON_PACKAGES+="fsspec==2024.6.1 " # Install pydot for visq -${VENV_PYTHON} -m pip ${PIP_OPTIONS} install pydot==${VER_PYDOT} +PYTHON_PACKAGES+="pydot==${VER_PYDOT} " + +# Install numpy 1.x +PYTHON_PACKAGES+="numpy==${VER_NUMPY} " + +${VENV_PYTHON} -m pip ${PIP_OPTIONS} install --upgrade ${PYTHON_PACKAGES} ${TORCH_SOURCE_OPTION} diff --git a/compiler/one-cmds/one-prepare-venv.u2404 b/compiler/one-cmds/one-prepare-venv.u2404 index 05ef0a19cdb..b5835edec25 100644 --- a/compiler/one-cmds/one-prepare-venv.u2404 +++ b/compiler/one-cmds/one-prepare-venv.u2404 @@ -26,19 +26,19 @@ VENV_PYTHON=${DRIVER_PATH}/venv/bin/python if [ ! -f ${VENV_ACTIVATE} ]; then # Create python virtual enviornment - python3.8 -m venv "${DRIVER_PATH}/venv" + python3 -m venv "${DRIVER_PATH}/venv" fi # NOTE version # - https://github.com/onnx/onnx/blob/master/docs/Versioning.md # - https://github.com/onnx/onnx-tensorflow/blob/master/Versioning.md -VER_TENSORFLOW=2.12.1 -VER_ONNX=1.16.0 -VER_ONNXRUNTIME=1.18.0 -VER_ONNX_TF=1.10.0 +VER_TENSORFLOW=2.19.0 # 2025-03-12 +VER_ONNX=1.17.0 +VER_ONNXRUNTIME=1.20.1 +#VER_ONNX_TF=1.10.0 VER_PYDOT=1.4.2 -VER_TORCH="2.1.2+cpu" +VER_TORCH="2.6.0+cpu" # Install tensorflow @@ -66,8 +66,8 @@ else fi ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install Pillow # Fix version to that of TF release date -${VENV_PYTHON} -m pip ${PIP_OPTIONS} install tensorflow_probability==0.20.1 -${VENV_PYTHON} -m pip ${PIP_OPTIONS} install tensorflow_addons==0.20.0 +#${VENV_PYTHON} -m pip ${PIP_OPTIONS} install tensorflow_probability==0.25.0 +#${VENV_PYTHON} -m pip ${PIP_OPTIONS} install tensorflow_addons==0.23.0 # Install PyTorch and ONNX related # NOTE set ONE_PREPVENV_TORCH_SOURCE to override options for source URL. @@ -84,14 +84,14 @@ ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install onnx==${VER_ONNX} ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install onnxruntime==${VER_ONNXRUNTIME} # Provide install of custom onnx-tf -if [ -n "${EXT_ONNX_TF_WHL}" ]; then - ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install ${EXT_ONNX_TF_WHL} -else - ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install onnx-tf==${VER_ONNX_TF} -fi +#if [ -n "${EXT_ONNX_TF_WHL}" ]; then +# ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install ${EXT_ONNX_TF_WHL} +#else +# ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install onnx-tf==${VER_ONNX_TF} +#fi # Fix version to that of TF release date -${VENV_PYTHON} -m pip ${PIP_OPTIONS} install --upgrade protobuf==4.23.3 +${VENV_PYTHON} -m pip ${PIP_OPTIONS} install --upgrade protobuf==5.29.4 # Install pydot for visq ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install pydot==${VER_PYDOT} diff --git a/compiler/one-cmds/tests/CMakeLists.txt b/compiler/one-cmds/tests/CMakeLists.txt index 9f343a9f114..5374722bc7a 100644 --- a/compiler/one-cmds/tests/CMakeLists.txt +++ b/compiler/one-cmds/tests/CMakeLists.txt @@ -78,6 +78,7 @@ file(APPEND "${DRIVER_SCRIPT}" else echo \"ALL TESTS PASSED!\" fi\n +echo \"Test results are also recorded in runtestall.log file.\" ") set(PREPARE_TEST_MATERIALS_SH "${CMAKE_CURRENT_SOURCE_DIR}/prepare_test_materials.sh") diff --git a/infra/nncc/CMakeLists.txt b/infra/nncc/CMakeLists.txt index 6eebcd4a4b4..e3c669c1ed1 100644 --- a/infra/nncc/CMakeLists.txt +++ b/infra/nncc/CMakeLists.txt @@ -116,8 +116,7 @@ include("cmake/ApplyCompileFlags.cmake") ### add_library(nncc_common INTERFACE) if(ENABLE_STRICT_BUILD) - # TODO Remove -Wno-overloaded-virtual - target_compile_options(nncc_common INTERFACE -Werror -Wall -Wextra -Wno-overloaded-virtual) + target_compile_options(nncc_common INTERFACE -Werror -Wall -Wextra) endif(ENABLE_STRICT_BUILD) add_library(nncc_coverage INTERFACE)