From ba03878cd4891fffde34e4e9a73ce2db779ad00e Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Fri, 28 Apr 2023 22:03:05 +0000 Subject: [PATCH 1/3] remove features --- rbc/irtools.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rbc/irtools.py b/rbc/irtools.py index 1c7c6070..9628bd6b 100644 --- a/rbc/irtools.py +++ b/rbc/irtools.py @@ -146,6 +146,8 @@ def _get_host_cpu_features(self): # See https://github.com/xnd-project/rbc/issues/45 remove_features = { (12, 12): [], (11, 11): [], (10, 10): [], (9, 9): [], (8, 8): [], + (14, 11): ['crc32', 'uintr', 'widekl', 'avxvnni', 'avx512fp16', 'kl', + 'hreset'], (11, 8): ['tsxldtrk', 'amx-tile', 'amx-bf16', 'serialize', 'amx-int8', 'avx512vp2intersect', 'tsxldtrk', 'amx-tile', 'amx-bf16', 'serialize', 'amx-int8', 'avx512vp2intersect', 'tsxldtrk', From fe654056fe82252f987b81ae35a45cd98c655c91 Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Mon, 8 May 2023 22:40:06 -0300 Subject: [PATCH 2/3] raise error on LLVM version mismatch --- rbc/irtools.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/rbc/irtools.py b/rbc/irtools.py index 9628bd6b..14141d6b 100644 --- a/rbc/irtools.py +++ b/rbc/irtools.py @@ -227,6 +227,10 @@ def post_lowering(self, mod, library): # Code generation methods +class LLVMVersionMismatchError(Exception): + pass + + @contextmanager def replace_numba_internals_hack(): # Hackish solution to prevent numba from calling _ensure_finalize. See issue #87 @@ -440,6 +444,27 @@ def compile_to_LLVM(functions_and_signatures, LLVM module instance. To get the IR string, use `str(module)`. """ + # check LLVM version before compiling to LLVM + server_llvm_version = target_info.llvm_version + client_llvm_version = llvm.llvm_version_info + + if (server_llvm_version[0], client_llvm_version[0]) == (11, 14): + c_llvm = '.'.join(map(str, client_llvm_version)) + s_llvm = '.'.join(map(str, server_llvm_version)) + flag = 'RBC_DISABLE_LLVM_MISMATCH_ERROR' + msg = (f'The client LLVM version ({c_llvm}) is greater than the server ' + f'LLVM version ({s_llvm}). This is known to be unsupported. ' + 'Please, downgrade to a previous release of Numba that uses the ' + 'same LLVM version as the HeavyDB server. For more information, ' + 'see the table below:\n\n' + 'https://github.com/numba/llvmlite#compatibility\n' + 'https://github.com/heavyai/heavydb#dependencies\n\n' + f'To disable this error, run RBC with {flag}=1 flag enabled.') + + DISABLE_LLVM_MISMATCH_ERROR = int(os.environ.get(flag, False)) + if not DISABLE_LLVM_MISMATCH_ERROR: + raise LLVMVersionMismatchError(msg) + target_desc = registry.cpu_target typing_context = JITRemoteTypingContext() From 12c57a16d3344a05ceb642c29397891f11fa3cc8 Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Tue, 9 May 2023 14:43:50 -0300 Subject: [PATCH 3/3] add test for numba/heavydb llvm mismatch --- rbc/errors.py | 15 ++++++++--- rbc/irtools.py | 7 +---- rbc/tests/heavydb/test_heavydb.py | 45 ++++++++++++++++++++++++++++--- 3 files changed, 54 insertions(+), 13 deletions(-) diff --git a/rbc/errors.py b/rbc/errors.py index 9b91b3fb..09606505 100644 --- a/rbc/errors.py +++ b/rbc/errors.py @@ -3,9 +3,18 @@ """ -from rbc.utils import get_version from numba.core.errors import TypingError +from rbc.utils import get_version + + +class LLVMVersionMismatchError(Exception): + """ + Raised when Numba and HeavyDB uses different LLVM version which is known to + be incompatible/problematic. + """ + pass + class HeavyDBServerError(Exception): """ @@ -55,5 +64,5 @@ class NumbaNotImplementedError(TypingError): class RequireLiteralValue(TypingError): pass else: - from numba.core.errors import NumbaTypeError, NumbaNotImplementedError, \ - RequireLiteralValue # noqa: F401 + from numba.core.errors import RequireLiteralValue # noqa: F401 + from numba.core.errors import NumbaNotImplementedError, NumbaTypeError diff --git a/rbc/irtools.py b/rbc/irtools.py index 14141d6b..8656fd1f 100644 --- a/rbc/irtools.py +++ b/rbc/irtools.py @@ -18,7 +18,7 @@ from rbc.externals import stdio from rbc.nrt import create_nrt_functions -from .errors import UnsupportedError +from .errors import UnsupportedError, LLVMVersionMismatchError from .libfuncs import Library from .targetinfo import TargetInfo @@ -226,11 +226,6 @@ def post_lowering(self, mod, library): # --------------------------------------------------------------------------- # Code generation methods - -class LLVMVersionMismatchError(Exception): - pass - - @contextmanager def replace_numba_internals_hack(): # Hackish solution to prevent numba from calling _ensure_finalize. See issue #87 diff --git a/rbc/tests/heavydb/test_heavydb.py b/rbc/tests/heavydb/test_heavydb.py index eb48dfca..41b1bda4 100644 --- a/rbc/tests/heavydb/test_heavydb.py +++ b/rbc/tests/heavydb/test_heavydb.py @@ -1,10 +1,14 @@ -import os import itertools -import pytest +import os + +import llvmlite.binding as llvm import numpy as np +import pytest -from rbc.errors import UnsupportedError, HeavyDBServerError -from rbc.tests import heavydb_fixture, assert_equal +from rbc.errors import (HeavyDBServerError, LLVMVersionMismatchError, + UnsupportedError) +from rbc.targetinfo import TargetInfo +from rbc.tests import assert_equal, heavydb_fixture from rbc.typesystem import Type rbc_heavydb = pytest.importorskip('rbc.heavydb') @@ -872,3 +876,36 @@ def rbc_test_non_admin_user_udf(x): # clean up: heavydb.sql_execute(f'DROP DATABASE IF EXISTS {dbname};') heavydb.sql_execute(f'DROP USER IF EXISTS "{user}";') + + +@pytest.mark.parametrize('kind', ('udf', 'udtf')) +def test_numba_heavydb_llvm_mismatch(heavydb, kind): + heavydb.reset() + + # only run this test on a specific environment + target_info = TargetInfo() + server_llvm_version = target_info.llvm_version + client_llvm_version = llvm.llvm_version_info + + if (server_llvm_version[0], client_llvm_version[0]) != (11, 14): + c_llvm = '.'.join(map(str, client_llvm_version)) + s_llvm = '.'.join(map(str, server_llvm_version)) + msg = (f'Test requires server LLVM 14, got {s_llvm}. And client LLVM ' + f'11, got {c_llvm}') + pytest.skip(msg) + + if kind == 'udf': + @heavydb('int32(int32)') + def add(a): + return a + 1 + else: + @heavydb('int32(TableFunctionManager, Column, OutputColumn)') + def column_copy(mgr, inp, out): + size = len(inp) + mgr.set_output_row_size(size) + for i in range(size): + out[i] = inp[i] + return size + + with pytest.raises(LLVMVersionMismatchError): + heavydb.register()