From 2abf0d58149d29fd942d5741ce046a2a296b6410 Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Tue, 25 Apr 2023 11:21:46 -0300 Subject: [PATCH 01/26] Update Numba to 0.57 --- .github/workflows/rbc_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rbc_test.yml b/.github/workflows/rbc_test.yml index 3b8a9e7f1..15e770753 100644 --- a/.github/workflows/rbc_test.yml +++ b/.github/workflows/rbc_test.yml @@ -108,8 +108,8 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - python-version: ['3.10', '3.9', '3.8'] - numba-version: ['0.56', '0.55'] + python-version: ['3.11', '3.10', '3.9'] + numba-version: ['0.57', '0.56'] heavydb-version: ['6.2', '6.1', '6.0'] heavydb-from: [conda] # include: From 84d2ca2a7e4fa72f0edd57ef24f3d6957031fbfb Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Tue, 2 May 2023 19:31:58 +0000 Subject: [PATCH 02/26] Fix a bug where returned structs were being allocated on the stack. Replace llvm alloca by allocate_varlen_buffer --- rbc/heavydb/array.py | 9 ++++---- rbc/heavydb/buffer.py | 27 ++++++++++++++--------- rbc/libfuncs.py | 1 + rbc/stdlib/creation_functions.py | 2 +- rbc/tests/heavydb/test_array_functions.py | 2 +- 5 files changed, 24 insertions(+), 17 deletions(-) diff --git a/rbc/heavydb/array.py b/rbc/heavydb/array.py index 14aca8f0a..bf282a3aa 100644 --- a/rbc/heavydb/array.py +++ b/rbc/heavydb/array.py @@ -46,12 +46,11 @@ def deepcopy(self, context, builder, val, retptr): ptr_type = self.dtype.members[0] element_size = int64_t(ptr_type.dtype.bitwidth // 8) - struct_load = builder.load(val, name='struct_load') - src = builder.extract_value(struct_load, 0, name='array_buff_ptr') - element_count = builder.extract_value(struct_load, 1, name='array_size') - is_null = builder.extract_value(struct_load, 2, name='array_is_null') - zero, one, two = int32_t(0), int32_t(1), int32_t(2) + src = builder.load(builder.gep(val, [zero, zero]), name='array_buff_ptr') + element_count = builder.load(builder.gep(val, [zero, one]), name='array_size') + is_null = builder.load(builder.gep(val, [zero, two]), name='array_is_null') + with builder.if_else(cgutils.is_true(builder, is_null)) as (then, otherwise): with then: nullptr = cgutils.get_null_value(src.type) diff --git a/rbc/heavydb/buffer.py b/rbc/heavydb/buffer.py index 0e3e6e81f..973b1bb1a 100644 --- a/rbc/heavydb/buffer.py +++ b/rbc/heavydb/buffer.py @@ -24,7 +24,9 @@ import operator - +from .allocator import allocate_varlen_buffer +from .metatype import HeavyDBMetaType +from llvmlite import ir import numpy as np from llvmlite import ir from numba.core import cgutils, datamodel, extending, imputils, types @@ -302,12 +304,16 @@ def heavydb_buffer_ptr_len_(typingctx, data): sig = types.int64(data) def codegen(context, builder, signature, args): - data, = args - rawptr = cgutils.alloca_once_value(builder, value=data) - struct = builder.load(builder.gep(rawptr, - [int32_t(0)])) - return builder.load(builder.gep( - struct, [int32_t(0), int32_t(1)])) + i32 = ir.IntType(32) + zero, one = i32(0), i32(1) + [data] = args + return builder.load(builder.gep(data, [zero, one])) + # data, = args + # rawptr = cgutils.alloca_once_value(builder, value=data) + # struct = builder.load(builder.gep(rawptr, + # [int32_t(0)])) + # return builder.load(builder.gep( + # struct, [int32_t(0), int32_t(1)])) return sig, codegen @@ -377,10 +383,11 @@ def codegen(context, builder, sig, args): data, index, value = args - rawptr = cgutils.alloca_once_value(builder, value=data) - ptr = builder.load(rawptr) + # breakpoint() + # rawptr = cgutils.alloca_once_value(builder, value=data) + # ptr = builder.load(rawptr) - buf = builder.load(builder.gep(ptr, [zero, zero])) + buf = builder.load(builder.gep(data, [zero, zero])) # [rbc issue-197] Numba promotes operations like # int32(a) + int32(b) to int64 fromty = sig.args[2] diff --git a/rbc/libfuncs.py b/rbc/libfuncs.py index de88f044b..191745020 100644 --- a/rbc/libfuncs.py +++ b/rbc/libfuncs.py @@ -282,6 +282,7 @@ def check(self, fname): experimental.vector.reduce.smin.* experimental.vector.reduce.umax.* experimental.vector.reduce.umin.* experimental.vector.reduce.fmax.* experimental.vector.reduce.fmin.* + experimental.noalias.scope.decl flt.rounds var.annotation ptr.annotation annotation codeview.annotation trap debugtrap stackprotector stackguard objectsize expect expect.with.probability assume ssa_copy diff --git a/rbc/stdlib/creation_functions.py b/rbc/stdlib/creation_functions.py index 2f886ea80..71faad2cb 100644 --- a/rbc/stdlib/creation_functions.py +++ b/rbc/stdlib/creation_functions.py @@ -105,7 +105,7 @@ def _array_api_triu(x, k=0): @expose.implements('full') -def _impl__full(shape, fill_value, dtype=None): +def _impl_full(shape, fill_value, dtype=None): """ Return a new array of given shape and type, filled with fill_value. """ diff --git a/rbc/tests/heavydb/test_array_functions.py b/rbc/tests/heavydb/test_array_functions.py index 848cc2bc6..8521c2ce1 100644 --- a/rbc/tests/heavydb/test_array_functions.py +++ b/rbc/tests/heavydb/test_array_functions.py @@ -155,7 +155,7 @@ def test_array_methods(heavydb, method, signature, args, expected): query = 'select np_{method}'.format(**locals()) + \ '(' + ', '.join(map(str, args)) + ')' + \ - ' from {heavydb.table_name};'.format(**locals()) + ' from {heavydb.table_name} limit 1;'.format(**locals()) _, result = heavydb.sql_execute(query) out = list(result)[0] From be0833d65d64afcddf5a32e811bf0077c69e16e2 Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Tue, 2 May 2023 21:18:47 -0300 Subject: [PATCH 03/26] undo changes --- rbc/heavydb/buffer.py | 29 ++++++++--------------- rbc/tests/heavydb/test_array_functions.py | 2 +- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/rbc/heavydb/buffer.py b/rbc/heavydb/buffer.py index 973b1bb1a..e8167746e 100644 --- a/rbc/heavydb/buffer.py +++ b/rbc/heavydb/buffer.py @@ -24,7 +24,6 @@ import operator -from .allocator import allocate_varlen_buffer from .metatype import HeavyDBMetaType from llvmlite import ir import numpy as np @@ -212,10 +211,7 @@ def heavydb_buffer_constructor(context, builder, sig, args): ptr = memalloc(context, builder, ptr_type, element_count, element_size) llty = context.get_value_type(sig.return_type.dtype) - st_size = context.get_abi_sizeof(llty) - st_ptr = builder.bitcast(allocate_varlen_buffer(builder, int64_t(st_size), - int64_t(1)), - llty.as_pointer()) + st_ptr = builder.alloca(llty) zero, one, two = int32_t(0), int32_t(1), int32_t(2) builder.store(ptr, builder.gep(st_ptr, [zero, zero])) @@ -304,16 +300,12 @@ def heavydb_buffer_ptr_len_(typingctx, data): sig = types.int64(data) def codegen(context, builder, signature, args): - i32 = ir.IntType(32) - zero, one = i32(0), i32(1) - [data] = args - return builder.load(builder.gep(data, [zero, one])) - # data, = args - # rawptr = cgutils.alloca_once_value(builder, value=data) - # struct = builder.load(builder.gep(rawptr, - # [int32_t(0)])) - # return builder.load(builder.gep( - # struct, [int32_t(0), int32_t(1)])) + data, = args + rawptr = cgutils.alloca_once_value(builder, value=data) + struct = builder.load(builder.gep(rawptr, + [int32_t(0)])) + return builder.load(builder.gep( + struct, [int32_t(0), int32_t(1)])) return sig, codegen @@ -383,11 +375,10 @@ def codegen(context, builder, sig, args): data, index, value = args - # breakpoint() - # rawptr = cgutils.alloca_once_value(builder, value=data) - # ptr = builder.load(rawptr) + rawptr = cgutils.alloca_once_value(builder, value=data) + ptr = builder.load(rawptr) - buf = builder.load(builder.gep(data, [zero, zero])) + buf = builder.load(builder.gep(ptr, [zero, zero])) # [rbc issue-197] Numba promotes operations like # int32(a) + int32(b) to int64 fromty = sig.args[2] diff --git a/rbc/tests/heavydb/test_array_functions.py b/rbc/tests/heavydb/test_array_functions.py index 8521c2ce1..848cc2bc6 100644 --- a/rbc/tests/heavydb/test_array_functions.py +++ b/rbc/tests/heavydb/test_array_functions.py @@ -155,7 +155,7 @@ def test_array_methods(heavydb, method, signature, args, expected): query = 'select np_{method}'.format(**locals()) + \ '(' + ', '.join(map(str, args)) + ')' + \ - ' from {heavydb.table_name} limit 1;'.format(**locals()) + ' from {heavydb.table_name};'.format(**locals()) _, result = heavydb.sql_execute(query) out = list(result)[0] From f88caeab1379847900c1583291a3b22520180100 Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Fri, 5 May 2023 17:09:53 -0300 Subject: [PATCH 04/26] Install llvm-14 with Numba 0.57 and llvm 11 with Numba 0.56 --- .github/workflows/rbc_test.yml | 9 +++++++++ environment.yml | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rbc_test.yml b/.github/workflows/rbc_test.yml index 15e770753..2fe2cb603 100644 --- a/.github/workflows/rbc_test.yml +++ b/.github/workflows/rbc_test.yml @@ -111,7 +111,13 @@ jobs: python-version: ['3.11', '3.10', '3.9'] numba-version: ['0.57', '0.56'] heavydb-version: ['6.2', '6.1', '6.0'] + llvm-version: ['11.1', '14'] heavydb-from: [conda] + exclude: + - numba-version: '0.56' + llvm-version: '14' + - numba-version: '0.57' + llvm-version: '11.1' # include: # - os: ubuntu-latest # python-version: '3.10' @@ -197,6 +203,9 @@ jobs: cat environment.yml > rbc_test.yaml echo " - numba=${{ matrix.numba-version }}" >> rbc_test.yaml echo " - python=${{ matrix.python-version }}" >> rbc_test.yaml + echo " - clang=${{ matrix.llvm-version }}" >> rbc_test.yaml + echo " - clangxx=${{ matrix.llvm-version }}" >> rbc_test.yaml + mamba env create --file=rbc_test.yaml -n rbc - name: rbc conda config diff --git a/environment.yml b/environment.yml index c8ef93b59..7667266ff 100644 --- a/environment.yml +++ b/environment.yml @@ -14,8 +14,8 @@ dependencies: - setuptools - numba>=0.55 - llvmlite - - clang==11.1 - - clangxx==11.1 + - clang + - clangxx - pytest - packaging - tblib From 738da48e198eb96aafb5a968da3f0aecba906d8b Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Tue, 9 May 2023 15:00:04 -0300 Subject: [PATCH 05/26] exclude numba 0.56 / python 3.11 from the matrix --- .github/workflows/rbc_test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/rbc_test.yml b/.github/workflows/rbc_test.yml index 2fe2cb603..f394a03a3 100644 --- a/.github/workflows/rbc_test.yml +++ b/.github/workflows/rbc_test.yml @@ -116,6 +116,8 @@ jobs: exclude: - numba-version: '0.56' llvm-version: '14' + - numba-version: '0.56' + python-version: '3.11' - numba-version: '0.57' llvm-version: '11.1' # include: From 157a97c4290fa93169ec0ce555693d02cb0fb3fc Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Tue, 9 May 2023 15:45:51 -0300 Subject: [PATCH 06/26] refactor matrix --- .github/workflows/rbc_test.yml | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/rbc_test.yml b/.github/workflows/rbc_test.yml index f394a03a3..d25d281c2 100644 --- a/.github/workflows/rbc_test.yml +++ b/.github/workflows/rbc_test.yml @@ -108,19 +108,15 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - python-version: ['3.11', '3.10', '3.9'] - numba-version: ['0.57', '0.56'] + python-version: ['3.10', '3.9'] + numba-version: ['0.56'] heavydb-version: ['6.2', '6.1', '6.0'] - llvm-version: ['11.1', '14'] + llvm-version: ['11.1'] heavydb-from: [conda] - exclude: - - numba-version: '0.56' + include: + - python-version: '3.11' + numba-version: '0.57' llvm-version: '14' - - numba-version: '0.56' - python-version: '3.11' - - numba-version: '0.57' - llvm-version: '11.1' - # include: # - os: ubuntu-latest # python-version: '3.10' # numba-version: '0.56' From 048a921ffb49bed11989bbaea029a75498248c84 Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Tue, 9 May 2023 16:01:15 -0300 Subject: [PATCH 07/26] fix rbc_test.yaml --- .github/workflows/rbc_test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/rbc_test.yml b/.github/workflows/rbc_test.yml index d25d281c2..dc510075f 100644 --- a/.github/workflows/rbc_test.yml +++ b/.github/workflows/rbc_test.yml @@ -114,8 +114,11 @@ jobs: llvm-version: ['11.1'] heavydb-from: [conda] include: - - python-version: '3.11' + - os: ubuntu-latest + python-version: '3.11' numba-version: '0.57' + heavydb-version: '6.2' + heavydb-from: conda llvm-version: '14' # - os: ubuntu-latest # python-version: '3.10' From b7286cb326fadb29c3e6d2ae77a9ec3f2b507f68 Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Tue, 9 May 2023 16:07:08 -0300 Subject: [PATCH 08/26] attempt to install numba from numba channel --- .github/workflows/rbc_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rbc_test.yml b/.github/workflows/rbc_test.yml index dc510075f..209ebe4d9 100644 --- a/.github/workflows/rbc_test.yml +++ b/.github/workflows/rbc_test.yml @@ -207,7 +207,7 @@ jobs: echo " - clang=${{ matrix.llvm-version }}" >> rbc_test.yaml echo " - clangxx=${{ matrix.llvm-version }}" >> rbc_test.yaml - mamba env create --file=rbc_test.yaml -n rbc + mamba env create --file=rbc_test.yaml -c conda-forge -c numba -n rbc - name: rbc conda config shell: bash -l {0} From 700e9537c459eaeba1cd1f8de9ee5cca25599876 Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Tue, 9 May 2023 16:17:26 -0300 Subject: [PATCH 09/26] add numba channel --- .github/workflows/rbc_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rbc_test.yml b/.github/workflows/rbc_test.yml index 209ebe4d9..d2d8f1ada 100644 --- a/.github/workflows/rbc_test.yml +++ b/.github/workflows/rbc_test.yml @@ -148,7 +148,7 @@ jobs: with: python-version: ${{ matrix.python-version }} channel-priority: strict - channels: conda-forge + channels: conda-forge,numba - name: Install mamba shell: bash -l {0} @@ -207,7 +207,7 @@ jobs: echo " - clang=${{ matrix.llvm-version }}" >> rbc_test.yaml echo " - clangxx=${{ matrix.llvm-version }}" >> rbc_test.yaml - mamba env create --file=rbc_test.yaml -c conda-forge -c numba -n rbc + mamba env create --file=rbc_test.yaml -n rbc - name: rbc conda config shell: bash -l {0} From 935bebf9a0d820e8786b221e40fb50bf48d037a9 Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Tue, 9 May 2023 16:25:25 -0300 Subject: [PATCH 10/26] replace strict priority by flexible --- .github/workflows/rbc_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rbc_test.yml b/.github/workflows/rbc_test.yml index d2d8f1ada..f45da4330 100644 --- a/.github/workflows/rbc_test.yml +++ b/.github/workflows/rbc_test.yml @@ -147,7 +147,7 @@ jobs: - uses: conda-incubator/setup-miniconda@v2 with: python-version: ${{ matrix.python-version }} - channel-priority: strict + channel-priority: flexible channels: conda-forge,numba - name: Install mamba From 2f01d508bca3eacdc1c39136de7d082aae74f912 Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Fri, 28 Apr 2023 22:03:05 +0000 Subject: [PATCH 11/26] remove features --- rbc/irtools.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rbc/irtools.py b/rbc/irtools.py index b8b99ebfa..52315ca8c 100644 --- a/rbc/irtools.py +++ b/rbc/irtools.py @@ -146,6 +146,8 @@ def _get_host_cpu_features(self): # See https://github.com/xnd-project/rbc/issues/45 remove_features = { (12, 12): [], (11, 11): [], (10, 10): [], (9, 9): [], (8, 8): [], + (14, 11): ['crc32', 'uintr', 'widekl', 'avxvnni', 'avx512fp16', 'kl', + 'hreset'], (11, 8): ['tsxldtrk', 'amx-tile', 'amx-bf16', 'serialize', 'amx-int8', 'avx512vp2intersect', 'tsxldtrk', 'amx-tile', 'amx-bf16', 'serialize', 'amx-int8', 'avx512vp2intersect', 'tsxldtrk', From 8a474352ef064893fbdced7219f11c38eb471f10 Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Mon, 8 May 2023 22:40:06 -0300 Subject: [PATCH 12/26] raise error on LLVM version mismatch --- rbc/irtools.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/rbc/irtools.py b/rbc/irtools.py index 52315ca8c..8e06518fd 100644 --- a/rbc/irtools.py +++ b/rbc/irtools.py @@ -228,6 +228,10 @@ def post_lowering(self, mod, library): # Code generation methods +class LLVMVersionMismatchError(Exception): + pass + + @contextmanager def replace_numba_internals_hack(): # Hackish solution to prevent numba from calling _ensure_finalize. See issue #87 @@ -441,6 +445,27 @@ def compile_to_LLVM(functions_and_signatures, LLVM module instance. To get the IR string, use `str(module)`. """ + # check LLVM version before compiling to LLVM + server_llvm_version = target_info.llvm_version + client_llvm_version = llvm.llvm_version_info + + if (server_llvm_version[0], client_llvm_version[0]) == (11, 14): + c_llvm = '.'.join(map(str, client_llvm_version)) + s_llvm = '.'.join(map(str, server_llvm_version)) + flag = 'RBC_DISABLE_LLVM_MISMATCH_ERROR' + msg = (f'The client LLVM version ({c_llvm}) is greater than the server ' + f'LLVM version ({s_llvm}). This is known to be unsupported. ' + 'Please, downgrade to a previous release of Numba that uses the ' + 'same LLVM version as the HeavyDB server. For more information, ' + 'see the table below:\n\n' + 'https://github.com/numba/llvmlite#compatibility\n' + 'https://github.com/heavyai/heavydb#dependencies\n\n' + f'To disable this error, run RBC with {flag}=1 flag enabled.') + + DISABLE_LLVM_MISMATCH_ERROR = int(os.environ.get(flag, False)) + if not DISABLE_LLVM_MISMATCH_ERROR: + raise LLVMVersionMismatchError(msg) + target_desc = registry.cpu_target typing_context = JITRemoteTypingContext() From 3648591ea5dae31d7f235446f4b7cec5674159cd Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Tue, 9 May 2023 14:43:50 -0300 Subject: [PATCH 13/26] add test for numba/heavydb llvm mismatch --- rbc/errors.py | 15 ++++++++--- rbc/irtools.py | 7 +---- rbc/tests/heavydb/test_heavydb.py | 45 ++++++++++++++++++++++++++++--- 3 files changed, 54 insertions(+), 13 deletions(-) diff --git a/rbc/errors.py b/rbc/errors.py index 9b91b3fbf..096065054 100644 --- a/rbc/errors.py +++ b/rbc/errors.py @@ -3,9 +3,18 @@ """ -from rbc.utils import get_version from numba.core.errors import TypingError +from rbc.utils import get_version + + +class LLVMVersionMismatchError(Exception): + """ + Raised when Numba and HeavyDB uses different LLVM version which is known to + be incompatible/problematic. + """ + pass + class HeavyDBServerError(Exception): """ @@ -55,5 +64,5 @@ class NumbaNotImplementedError(TypingError): class RequireLiteralValue(TypingError): pass else: - from numba.core.errors import NumbaTypeError, NumbaNotImplementedError, \ - RequireLiteralValue # noqa: F401 + from numba.core.errors import RequireLiteralValue # noqa: F401 + from numba.core.errors import NumbaNotImplementedError, NumbaTypeError diff --git a/rbc/irtools.py b/rbc/irtools.py index 8e06518fd..b46e8c35c 100644 --- a/rbc/irtools.py +++ b/rbc/irtools.py @@ -18,7 +18,7 @@ from rbc.externals import stdio from rbc.nrt import create_nrt_functions -from .errors import UnsupportedError +from .errors import UnsupportedError, LLVMVersionMismatchError from .libfuncs import Library from .targetinfo import TargetInfo @@ -227,11 +227,6 @@ def post_lowering(self, mod, library): # --------------------------------------------------------------------------- # Code generation methods - -class LLVMVersionMismatchError(Exception): - pass - - @contextmanager def replace_numba_internals_hack(): # Hackish solution to prevent numba from calling _ensure_finalize. See issue #87 diff --git a/rbc/tests/heavydb/test_heavydb.py b/rbc/tests/heavydb/test_heavydb.py index eb48dfcad..41b1bda47 100644 --- a/rbc/tests/heavydb/test_heavydb.py +++ b/rbc/tests/heavydb/test_heavydb.py @@ -1,10 +1,14 @@ -import os import itertools -import pytest +import os + +import llvmlite.binding as llvm import numpy as np +import pytest -from rbc.errors import UnsupportedError, HeavyDBServerError -from rbc.tests import heavydb_fixture, assert_equal +from rbc.errors import (HeavyDBServerError, LLVMVersionMismatchError, + UnsupportedError) +from rbc.targetinfo import TargetInfo +from rbc.tests import assert_equal, heavydb_fixture from rbc.typesystem import Type rbc_heavydb = pytest.importorskip('rbc.heavydb') @@ -872,3 +876,36 @@ def rbc_test_non_admin_user_udf(x): # clean up: heavydb.sql_execute(f'DROP DATABASE IF EXISTS {dbname};') heavydb.sql_execute(f'DROP USER IF EXISTS "{user}";') + + +@pytest.mark.parametrize('kind', ('udf', 'udtf')) +def test_numba_heavydb_llvm_mismatch(heavydb, kind): + heavydb.reset() + + # only run this test on a specific environment + target_info = TargetInfo() + server_llvm_version = target_info.llvm_version + client_llvm_version = llvm.llvm_version_info + + if (server_llvm_version[0], client_llvm_version[0]) != (11, 14): + c_llvm = '.'.join(map(str, client_llvm_version)) + s_llvm = '.'.join(map(str, server_llvm_version)) + msg = (f'Test requires server LLVM 14, got {s_llvm}. And client LLVM ' + f'11, got {c_llvm}') + pytest.skip(msg) + + if kind == 'udf': + @heavydb('int32(int32)') + def add(a): + return a + 1 + else: + @heavydb('int32(TableFunctionManager, Column, OutputColumn)') + def column_copy(mgr, inp, out): + size = len(inp) + mgr.set_output_row_size(size) + for i in range(size): + out[i] = inp[i] + return size + + with pytest.raises(LLVMVersionMismatchError): + heavydb.register() From ea6f6798f7f3ba965361b18cb9d0116c7d46f506 Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Tue, 9 May 2023 20:02:56 -0300 Subject: [PATCH 14/26] add CI job for LLVM mismatch --- .github/workflows/rbc_test.yml | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rbc_test.yml b/.github/workflows/rbc_test.yml index f45da4330..943d10a89 100644 --- a/.github/workflows/rbc_test.yml +++ b/.github/workflows/rbc_test.yml @@ -117,7 +117,14 @@ jobs: - os: ubuntu-latest python-version: '3.11' numba-version: '0.57' - heavydb-version: '6.2' + heavydb-version: '6.2' # if changing this value, update the condition + # on "Run rbc tests [LLVM mismatch]" + heavydb-from: conda + llvm-version: '14' + - os: ubuntu-latest + python-version: '3.11' + numba-version: '0.57' + heavydb-version: '7.0' # to be released heavydb-from: conda llvm-version: '14' # - os: ubuntu-latest @@ -239,9 +246,9 @@ jobs: pkill -f heavydb mamba run -n rbc pytest -sv -r A rbc/tests/ -x --ignore rbc/tests/heavydb - - name: Run rbc tests + - name: Run rbc tests [conda] shell: bash -l {0} - if: ${{ ! ( matrix.os == 'ubuntu-latest' && matrix.heavydb-from == 'docker' ) }} + if: matrix.heavydb-from == 'conda' && ! (matrix.numba-version == '0.57' && matrix.heavydb-version == '6.2') env: HEAVYDB_SOURCE: ${{ matrix.heavydb-from }} EXPECTED_PYTHON_VERSION: ${{ matrix.python-version }} @@ -253,6 +260,19 @@ jobs: pkill -f heavydb mamba run -n rbc pytest -sv -r A rbc/tests/ -x --ignore rbc/tests/heavydb + - name: Run rbc tests [LLVM mismatch] + shell: bash -l {0} + if: matrix.numba-version == '0.57' && matrix.heavydb-version == '6.2' + env: + HEAVYDB_SOURCE: ${{ matrix.heavydb-from }} + EXPECTED_PYTHON_VERSION: ${{ matrix.python-version }} + EXPECTED_HEAVYDB_VERSION: ${{ matrix.heavydb-version }} + EXPECTED_NUMBA_VERSION: ${{ matrix.numba-version }} + RBC_TESTS_FULL: TRUE + run: | + mamba run -n rbc pytest -sv -r A rbc/tests/heavydb/test_heavydb.py -k test_numba_heavydb_llvm_mismatch + + - name: Show Heavydb conda logs on failure [conda] shell: bash -l {0} if: failure() && matrix.os == 'ubuntu-latest' && matrix.heavydb-from == 'conda' From 30b550e6a1ea1ed77bd5bb8a573e123dc05bf0d2 Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Tue, 9 May 2023 20:06:08 -0300 Subject: [PATCH 15/26] flake8 --- rbc/errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rbc/errors.py b/rbc/errors.py index 096065054..7102b13e6 100644 --- a/rbc/errors.py +++ b/rbc/errors.py @@ -65,4 +65,4 @@ class RequireLiteralValue(TypingError): pass else: from numba.core.errors import RequireLiteralValue # noqa: F401 - from numba.core.errors import NumbaNotImplementedError, NumbaTypeError + from numba.core.errors import NumbaNotImplementedError, NumbaTypeError # noqa: F401 From bdcf176218f889b309e0ac4b64ee246e8a9fc0b1 Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Tue, 9 May 2023 20:19:28 -0300 Subject: [PATCH 16/26] retrieve target info from heavydb.targets --- rbc/tests/heavydb/test_heavydb.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rbc/tests/heavydb/test_heavydb.py b/rbc/tests/heavydb/test_heavydb.py index 41b1bda47..4e64c4a49 100644 --- a/rbc/tests/heavydb/test_heavydb.py +++ b/rbc/tests/heavydb/test_heavydb.py @@ -7,7 +7,6 @@ from rbc.errors import (HeavyDBServerError, LLVMVersionMismatchError, UnsupportedError) -from rbc.targetinfo import TargetInfo from rbc.tests import assert_equal, heavydb_fixture from rbc.typesystem import Type @@ -883,7 +882,7 @@ def test_numba_heavydb_llvm_mismatch(heavydb, kind): heavydb.reset() # only run this test on a specific environment - target_info = TargetInfo() + target_info = heavydb.targets['cpu'] server_llvm_version = target_info.llvm_version client_llvm_version = llvm.llvm_version_info From 31ab121c043eba1245afb40605f73c88a8753991 Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Tue, 9 May 2023 20:56:55 -0300 Subject: [PATCH 17/26] add llvm_version to remote jit target info --- .github/workflows/rbc_test.yml | 4 ++-- rbc/remotejit.py | 14 +++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/rbc_test.yml b/.github/workflows/rbc_test.yml index 943d10a89..84d92f587 100644 --- a/.github/workflows/rbc_test.yml +++ b/.github/workflows/rbc_test.yml @@ -320,11 +320,11 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.10 - uses: conda-incubator/setup-miniconda@v2 with: - python-version: 3.8 + python-version: 3.10 channel-priority: strict channels: conda-forge environment-file: environment.yml diff --git a/rbc/remotejit.py b/rbc/remotejit.py index 9e8282f64..cffa17c92 100644 --- a/rbc/remotejit.py +++ b/rbc/remotejit.py @@ -3,17 +3,20 @@ __all__ = ['RemoteJIT', 'Signature', 'Caller'] -import os +import ctypes import inspect +import os import warnings -import ctypes from collections import defaultdict + +import llvmlite.binding as llvm + from . import irtools from .errors import UnsupportedError -from .typesystem import Type, get_signature -from .thrift import Server, Dispatcher, dispatchermethod, Data, Client -from .utils import get_local_ip, UNSPECIFIED, validate_devices from .targetinfo import TargetInfo +from .thrift import Client, Data, Dispatcher, Server, dispatchermethod +from .typesystem import Type, get_signature +from .utils import UNSPECIFIED, get_local_ip, validate_devices def isfunctionlike(obj): @@ -990,6 +993,7 @@ def targets(self) -> dict: target_info = TargetInfo.host() target_info.set('has_numba', True) target_info.set('has_cpython', True) + target_info.set('llvm_version', llvm.llvm_version_info) return dict(cpu=target_info.tojson()) @dispatchermethod From 40b5d759a6b1b62811b3fb4586e13cf68eff3e7e Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Tue, 9 May 2023 20:59:35 -0300 Subject: [PATCH 18/26] use string quotes to specify python version --- .github/workflows/rbc_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rbc_test.yml b/.github/workflows/rbc_test.yml index 84d92f587..ce048b8d9 100644 --- a/.github/workflows/rbc_test.yml +++ b/.github/workflows/rbc_test.yml @@ -320,11 +320,11 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.10 + python-version: '3.10' - uses: conda-incubator/setup-miniconda@v2 with: - python-version: 3.10 + python-version: '3.10' channel-priority: strict channels: conda-forge environment-file: environment.yml From 5337fe165f3fe8bcab4fd5bbaea08b7cbc334b29 Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Tue, 9 May 2023 21:24:59 -0300 Subject: [PATCH 19/26] undo change --- rbc/errors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rbc/errors.py b/rbc/errors.py index 7102b13e6..8ee90c4b1 100644 --- a/rbc/errors.py +++ b/rbc/errors.py @@ -64,5 +64,5 @@ class NumbaNotImplementedError(TypingError): class RequireLiteralValue(TypingError): pass else: - from numba.core.errors import RequireLiteralValue # noqa: F401 - from numba.core.errors import NumbaNotImplementedError, NumbaTypeError # noqa: F401 + from numba.core.errors import NumbaTypeError, NumbaNotImplementedError, \ + RequireLiteralValue # noqa: F401 From 47177324b063034f00355557051782f081926fcd Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Tue, 9 May 2023 21:27:18 -0300 Subject: [PATCH 20/26] document DISABLE_LLVM_MISMATCH_ERROR on envvars.rst --- doc/envvars.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/envvars.rst b/doc/envvars.rst index 1bc34fb6d..1db67b46e 100644 --- a/doc/envvars.rst +++ b/doc/envvars.rst @@ -14,3 +14,9 @@ Debugging If set to non-zero, insert debug statements to our implementation of Numba Runtime (NRT) + +.. envvar:: DISABLE_LLVM_MISMATCH_ERROR + + If set to non-zero, RBC will not raise an error on Numba/HeavyDB LLVM + mismatch version. Numba 0.57 uses LLVM 14 and may produce IR that + older versions of HeavyDB (< 7.0) cannot read. From 11fe42a39156bc537bed87bcff2877a2f79aed20 Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Wed, 10 May 2023 11:44:46 -0300 Subject: [PATCH 21/26] replace exception by warning --- doc/envvars.rst | 4 ++-- environment.yml | 6 +++--- rbc/errors.py | 8 -------- rbc/heavydb/buffer.py | 6 +++++- rbc/heavydb/remoteheavydb.py | 29 +++++++++++++++++++++++++++ rbc/irtools.py | 39 ++++++++++++++++-------------------- rbc/warnings.py | 7 +++++++ 7 files changed, 63 insertions(+), 36 deletions(-) create mode 100644 rbc/warnings.py diff --git a/doc/envvars.rst b/doc/envvars.rst index 1db67b46e..28ccedefd 100644 --- a/doc/envvars.rst +++ b/doc/envvars.rst @@ -15,8 +15,8 @@ Debugging If set to non-zero, insert debug statements to our implementation of Numba Runtime (NRT) -.. envvar:: DISABLE_LLVM_MISMATCH_ERROR +.. envvar:: DISABLE_LLVM_MISMATCH_WARN - If set to non-zero, RBC will not raise an error on Numba/HeavyDB LLVM + If set to non-zero, RBC will not raise a warning on Numba/HeavyDB LLVM mismatch version. Numba 0.57 uses LLVM 14 and may produce IR that older versions of HeavyDB (< 7.0) cannot read. diff --git a/environment.yml b/environment.yml index 7667266ff..ee63dd00e 100644 --- a/environment.yml +++ b/environment.yml @@ -12,10 +12,10 @@ channels: dependencies: - python>=3.7 - setuptools - - numba>=0.55 + - numba>=0.56 - llvmlite - - clang - - clangxx + - clang==11 + - clangxx==11 - pytest - packaging - tblib diff --git a/rbc/errors.py b/rbc/errors.py index 8ee90c4b1..49d999671 100644 --- a/rbc/errors.py +++ b/rbc/errors.py @@ -8,14 +8,6 @@ from rbc.utils import get_version -class LLVMVersionMismatchError(Exception): - """ - Raised when Numba and HeavyDB uses different LLVM version which is known to - be incompatible/problematic. - """ - pass - - class HeavyDBServerError(Exception): """ Raised when HeavyDB server raises a runtime error that RBC knows diff --git a/rbc/heavydb/buffer.py b/rbc/heavydb/buffer.py index e8167746e..f1cc26b7f 100644 --- a/rbc/heavydb/buffer.py +++ b/rbc/heavydb/buffer.py @@ -24,6 +24,7 @@ import operator +from .allocator import allocate_varlen_buffer from .metatype import HeavyDBMetaType from llvmlite import ir import numpy as np @@ -211,7 +212,10 @@ def heavydb_buffer_constructor(context, builder, sig, args): ptr = memalloc(context, builder, ptr_type, element_count, element_size) llty = context.get_value_type(sig.return_type.dtype) - st_ptr = builder.alloca(llty) + st_size = context.get_abi_sizeof(llty) + st_ptr = builder.bitcast(allocate_varlen_buffer(builder, int64_t(st_size), + int64_t(1)), + llty.as_pointer()) zero, one, two = int32_t(0), int32_t(1), int32_t(2) builder.store(ptr, builder.gep(st_ptr, [zero, zero])) diff --git a/rbc/heavydb/remoteheavydb.py b/rbc/heavydb/remoteheavydb.py index 721944a78..1eed9771f 100644 --- a/rbc/heavydb/remoteheavydb.py +++ b/rbc/heavydb/remoteheavydb.py @@ -10,6 +10,7 @@ import configparser import numpy import textwrap +import llvmlite.binding as llvm from collections import defaultdict, namedtuple from rbc.remotejit import RemoteJIT, RemoteCallCapsule from rbc.thrift.utils import resolve_includes @@ -25,6 +26,7 @@ from rbc.targetinfo import TargetInfo from rbc.irtools import compile_to_LLVM from rbc.errors import ForbiddenNameError, HeavyDBServerError +from rbc.warnings import LLVMVersionMismatchWarning from rbc.utils import parse_version, version_date from rbc import ctools, typesystem @@ -1266,8 +1268,35 @@ def _make_udf(self, caller, orig_sig, sig): name + sig.mangling(), atypes, rtype, annotations) + def _check_llvm_version(self): + # check LLVM version before compiling to LLVM + flag_name = 'RBC_DISABLE_LLVM_MISMATCH_WARN' + flag = int(os.environ.get(flag_name, False)) + + if not flag: + target = self.targets['cpu'] + server_llvm_version = target.llvm_version + client_llvm_version = llvm.llvm_version_info + + if (server_llvm_version[0], client_llvm_version[0]) == (11, 14): + c_llvm = '.'.join(map(str, client_llvm_version)) + s_llvm = '.'.join(map(str, server_llvm_version)) + msg = ( + f'The client LLVM version ({c_llvm}) is greater than the server ' + f'LLVM version ({s_llvm}). This is known to be unsupported. ' + 'Please, downgrade to a previous release of Numba that uses the ' + 'same LLVM version as the HeavyDB server. For more information, ' + 'see the table below:\n\n' + 'https://github.com/numba/llvmlite#compatibility\n' + 'https://github.com/heavyai/heavydb#dependencies\n\n' + f'To suppress this warning, run RBC with {flag_name}=1 ' + 'flag enabled.') + warnings.warn(msg, LLVMVersionMismatchWarning) + def register(self): """Register caller cache to the server.""" + self._check_llvm_version() + with typesystem.Type.alias(**self.typesystem_aliases): return self._register() diff --git a/rbc/irtools.py b/rbc/irtools.py index b46e8c35c..f6ff752cc 100644 --- a/rbc/irtools.py +++ b/rbc/irtools.py @@ -18,7 +18,7 @@ from rbc.externals import stdio from rbc.nrt import create_nrt_functions -from .errors import UnsupportedError, LLVMVersionMismatchError +from .errors import UnsupportedError from .libfuncs import Library from .targetinfo import TargetInfo @@ -146,6 +146,22 @@ def _get_host_cpu_features(self): # See https://github.com/xnd-project/rbc/issues/45 remove_features = { (12, 12): [], (11, 11): [], (10, 10): [], (9, 9): [], (8, 8): [], + (14, 14): ['avx512pf', 'tsxldtrk', 'cx16', 'sahf', 'tbm', + 'avx512ifma', 'sha', 'crc32', 'fma4', 'vpclmulqdq', + 'prfchw', 'bmi2', 'cldemote', 'fsgsbase', 'ptwrite', + 'amxtile', 'uintr', 'gfni', 'popcnt', 'widekl', 'aes', + 'avx512bitalg', 'movdiri', 'xsaves', 'avx512er', 'avxvnni', + 'avx512fp16', 'avx512vnni', 'amxbf16', 'avx512vpopcntdq', + 'pconfig', 'clwb', 'avx512f', 'xsavec', 'clzero', 'pku', + 'mmx', 'lwp', 'rdpid', 'xop', 'rdseed', 'waitpkg', 'kl', + 'movdir64b', 'sse4a', 'avx512bw', 'clflushopt', 'xsave', + 'avx512vbmi2', '64bit', 'avx512vl', 'serialize', 'hreset', + 'invpcid', 'avx512cd', 'avx', 'vaes', 'avx512bf16', 'cx8', + 'fma', 'rtm', 'bmi', 'enqcmd', 'rdrnd', 'mwaitx', 'sse4.1', + 'sse4.2', 'avx2', 'fxsr', 'wbnoinvd', 'sse', 'lzcnt', 'pclmul', + 'prefetchwt1', 'f16c', 'ssse3', 'sgx', 'shstk', 'cmov', + 'avx512vbmi', 'amxint8', 'movbe', 'avx512vp2intersect', + 'xsaveopt', 'avx512dq', 'sse2', 'adx', 'sse3'], (14, 11): ['crc32', 'uintr', 'widekl', 'avxvnni', 'avx512fp16', 'kl', 'hreset'], (11, 8): ['tsxldtrk', 'amx-tile', 'amx-bf16', 'serialize', 'amx-int8', @@ -440,27 +456,6 @@ def compile_to_LLVM(functions_and_signatures, LLVM module instance. To get the IR string, use `str(module)`. """ - # check LLVM version before compiling to LLVM - server_llvm_version = target_info.llvm_version - client_llvm_version = llvm.llvm_version_info - - if (server_llvm_version[0], client_llvm_version[0]) == (11, 14): - c_llvm = '.'.join(map(str, client_llvm_version)) - s_llvm = '.'.join(map(str, server_llvm_version)) - flag = 'RBC_DISABLE_LLVM_MISMATCH_ERROR' - msg = (f'The client LLVM version ({c_llvm}) is greater than the server ' - f'LLVM version ({s_llvm}). This is known to be unsupported. ' - 'Please, downgrade to a previous release of Numba that uses the ' - 'same LLVM version as the HeavyDB server. For more information, ' - 'see the table below:\n\n' - 'https://github.com/numba/llvmlite#compatibility\n' - 'https://github.com/heavyai/heavydb#dependencies\n\n' - f'To disable this error, run RBC with {flag}=1 flag enabled.') - - DISABLE_LLVM_MISMATCH_ERROR = int(os.environ.get(flag, False)) - if not DISABLE_LLVM_MISMATCH_ERROR: - raise LLVMVersionMismatchError(msg) - target_desc = registry.cpu_target typing_context = JITRemoteTypingContext() diff --git a/rbc/warnings.py b/rbc/warnings.py new file mode 100644 index 000000000..f129a41eb --- /dev/null +++ b/rbc/warnings.py @@ -0,0 +1,7 @@ + +class LLVMVersionMismatchWarning(UserWarning): + """ + Raised when Numba and HeavyDB uses different LLVM version which is known to + be incompatible/problematic. + """ + pass From 1f73a5c0660effcc97c4d035418e0611d6af3bd5 Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Wed, 10 May 2023 12:19:12 -0300 Subject: [PATCH 22/26] update test and environment.yml --- environment.yml | 4 ++-- rbc/tests/heavydb/test_heavydb.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/environment.yml b/environment.yml index ee63dd00e..33204f2cb 100644 --- a/environment.yml +++ b/environment.yml @@ -14,8 +14,8 @@ dependencies: - setuptools - numba>=0.56 - llvmlite - - clang==11 - - clangxx==11 + - clang + - clangxx - pytest - packaging - tblib diff --git a/rbc/tests/heavydb/test_heavydb.py b/rbc/tests/heavydb/test_heavydb.py index 4e64c4a49..bfbdde78a 100644 --- a/rbc/tests/heavydb/test_heavydb.py +++ b/rbc/tests/heavydb/test_heavydb.py @@ -5,8 +5,8 @@ import numpy as np import pytest -from rbc.errors import (HeavyDBServerError, LLVMVersionMismatchError, - UnsupportedError) +from rbc.warnings import LLVMVersionMismatchWarning +from rbc.errors import HeavyDBServerError, UnsupportedError from rbc.tests import assert_equal, heavydb_fixture from rbc.typesystem import Type @@ -906,5 +906,5 @@ def column_copy(mgr, inp, out): out[i] = inp[i] return size - with pytest.raises(LLVMVersionMismatchError): + with pytest.warns(LLVMVersionMismatchWarning): heavydb.register() From 6c8ee445da5d68bb060d5a433474ce83663a7cde Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Fri, 12 May 2023 13:42:56 -0300 Subject: [PATCH 23/26] isort --- rbc/heavydb/buffer.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rbc/heavydb/buffer.py b/rbc/heavydb/buffer.py index f1cc26b7f..0e3e6e81f 100644 --- a/rbc/heavydb/buffer.py +++ b/rbc/heavydb/buffer.py @@ -24,9 +24,7 @@ import operator -from .allocator import allocate_varlen_buffer -from .metatype import HeavyDBMetaType -from llvmlite import ir + import numpy as np from llvmlite import ir from numba.core import cgutils, datamodel, extending, imputils, types From a91046121b3a67d7e1909115e39431f5563665d4 Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Fri, 12 May 2023 14:30:45 -0300 Subject: [PATCH 24/26] add heavyai 7.0.0 from docker --- .github/workflows/rbc_test.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rbc_test.yml b/.github/workflows/rbc_test.yml index ce048b8d9..43f3928d1 100644 --- a/.github/workflows/rbc_test.yml +++ b/.github/workflows/rbc_test.yml @@ -117,16 +117,17 @@ jobs: - os: ubuntu-latest python-version: '3.11' numba-version: '0.57' + llvm-version: '14' heavydb-version: '6.2' # if changing this value, update the condition # on "Run rbc tests [LLVM mismatch]" heavydb-from: conda - llvm-version: '14' - os: ubuntu-latest python-version: '3.11' numba-version: '0.57' - heavydb-version: '7.0' # to be released - heavydb-from: conda llvm-version: '14' + heavydb-version: '7.0' + docker-image: heavyai/heavyai-ee-cpu:7.0.0 + heavydb-from: docker # - os: ubuntu-latest # python-version: '3.10' # numba-version: '0.56' From 3f12d8f9c863913d6baabb188d6fdf6cd4b8fddb Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Mon, 12 Jun 2023 11:14:33 -0300 Subject: [PATCH 25/26] address reviewer comments --- .github/workflows/rbc_test.yml | 22 +------------------- doc/envvars.rst | 6 ------ rbc/heavydb/remoteheavydb.py | 28 ------------------------- rbc/irtools.py | 2 -- rbc/tests/heavydb/test_heavydb.py | 34 ------------------------------- rbc/warnings.py | 7 ------- 6 files changed, 1 insertion(+), 98 deletions(-) delete mode 100644 rbc/warnings.py diff --git a/.github/workflows/rbc_test.yml b/.github/workflows/rbc_test.yml index 43f3928d1..6fa7528db 100644 --- a/.github/workflows/rbc_test.yml +++ b/.github/workflows/rbc_test.yml @@ -114,13 +114,6 @@ jobs: llvm-version: ['11.1'] heavydb-from: [conda] include: - - os: ubuntu-latest - python-version: '3.11' - numba-version: '0.57' - llvm-version: '14' - heavydb-version: '6.2' # if changing this value, update the condition - # on "Run rbc tests [LLVM mismatch]" - heavydb-from: conda - os: ubuntu-latest python-version: '3.11' numba-version: '0.57' @@ -249,7 +242,7 @@ jobs: - name: Run rbc tests [conda] shell: bash -l {0} - if: matrix.heavydb-from == 'conda' && ! (matrix.numba-version == '0.57' && matrix.heavydb-version == '6.2') + if: matrix.heavydb-from == 'conda' env: HEAVYDB_SOURCE: ${{ matrix.heavydb-from }} EXPECTED_PYTHON_VERSION: ${{ matrix.python-version }} @@ -261,19 +254,6 @@ jobs: pkill -f heavydb mamba run -n rbc pytest -sv -r A rbc/tests/ -x --ignore rbc/tests/heavydb - - name: Run rbc tests [LLVM mismatch] - shell: bash -l {0} - if: matrix.numba-version == '0.57' && matrix.heavydb-version == '6.2' - env: - HEAVYDB_SOURCE: ${{ matrix.heavydb-from }} - EXPECTED_PYTHON_VERSION: ${{ matrix.python-version }} - EXPECTED_HEAVYDB_VERSION: ${{ matrix.heavydb-version }} - EXPECTED_NUMBA_VERSION: ${{ matrix.numba-version }} - RBC_TESTS_FULL: TRUE - run: | - mamba run -n rbc pytest -sv -r A rbc/tests/heavydb/test_heavydb.py -k test_numba_heavydb_llvm_mismatch - - - name: Show Heavydb conda logs on failure [conda] shell: bash -l {0} if: failure() && matrix.os == 'ubuntu-latest' && matrix.heavydb-from == 'conda' diff --git a/doc/envvars.rst b/doc/envvars.rst index 28ccedefd..1bc34fb6d 100644 --- a/doc/envvars.rst +++ b/doc/envvars.rst @@ -14,9 +14,3 @@ Debugging If set to non-zero, insert debug statements to our implementation of Numba Runtime (NRT) - -.. envvar:: DISABLE_LLVM_MISMATCH_WARN - - If set to non-zero, RBC will not raise a warning on Numba/HeavyDB LLVM - mismatch version. Numba 0.57 uses LLVM 14 and may produce IR that - older versions of HeavyDB (< 7.0) cannot read. diff --git a/rbc/heavydb/remoteheavydb.py b/rbc/heavydb/remoteheavydb.py index 1eed9771f..919bc7e47 100644 --- a/rbc/heavydb/remoteheavydb.py +++ b/rbc/heavydb/remoteheavydb.py @@ -26,7 +26,6 @@ from rbc.targetinfo import TargetInfo from rbc.irtools import compile_to_LLVM from rbc.errors import ForbiddenNameError, HeavyDBServerError -from rbc.warnings import LLVMVersionMismatchWarning from rbc.utils import parse_version, version_date from rbc import ctools, typesystem @@ -1268,35 +1267,8 @@ def _make_udf(self, caller, orig_sig, sig): name + sig.mangling(), atypes, rtype, annotations) - def _check_llvm_version(self): - # check LLVM version before compiling to LLVM - flag_name = 'RBC_DISABLE_LLVM_MISMATCH_WARN' - flag = int(os.environ.get(flag_name, False)) - - if not flag: - target = self.targets['cpu'] - server_llvm_version = target.llvm_version - client_llvm_version = llvm.llvm_version_info - - if (server_llvm_version[0], client_llvm_version[0]) == (11, 14): - c_llvm = '.'.join(map(str, client_llvm_version)) - s_llvm = '.'.join(map(str, server_llvm_version)) - msg = ( - f'The client LLVM version ({c_llvm}) is greater than the server ' - f'LLVM version ({s_llvm}). This is known to be unsupported. ' - 'Please, downgrade to a previous release of Numba that uses the ' - 'same LLVM version as the HeavyDB server. For more information, ' - 'see the table below:\n\n' - 'https://github.com/numba/llvmlite#compatibility\n' - 'https://github.com/heavyai/heavydb#dependencies\n\n' - f'To suppress this warning, run RBC with {flag_name}=1 ' - 'flag enabled.') - warnings.warn(msg, LLVMVersionMismatchWarning) - def register(self): """Register caller cache to the server.""" - self._check_llvm_version() - with typesystem.Type.alias(**self.typesystem_aliases): return self._register() diff --git a/rbc/irtools.py b/rbc/irtools.py index f6ff752cc..3cfb5665d 100644 --- a/rbc/irtools.py +++ b/rbc/irtools.py @@ -162,8 +162,6 @@ def _get_host_cpu_features(self): 'prefetchwt1', 'f16c', 'ssse3', 'sgx', 'shstk', 'cmov', 'avx512vbmi', 'amxint8', 'movbe', 'avx512vp2intersect', 'xsaveopt', 'avx512dq', 'sse2', 'adx', 'sse3'], - (14, 11): ['crc32', 'uintr', 'widekl', 'avxvnni', 'avx512fp16', 'kl', - 'hreset'], (11, 8): ['tsxldtrk', 'amx-tile', 'amx-bf16', 'serialize', 'amx-int8', 'avx512vp2intersect', 'tsxldtrk', 'amx-tile', 'amx-bf16', 'serialize', 'amx-int8', 'avx512vp2intersect', 'tsxldtrk', diff --git a/rbc/tests/heavydb/test_heavydb.py b/rbc/tests/heavydb/test_heavydb.py index bfbdde78a..f2248615f 100644 --- a/rbc/tests/heavydb/test_heavydb.py +++ b/rbc/tests/heavydb/test_heavydb.py @@ -5,7 +5,6 @@ import numpy as np import pytest -from rbc.warnings import LLVMVersionMismatchWarning from rbc.errors import HeavyDBServerError, UnsupportedError from rbc.tests import assert_equal, heavydb_fixture from rbc.typesystem import Type @@ -875,36 +874,3 @@ def rbc_test_non_admin_user_udf(x): # clean up: heavydb.sql_execute(f'DROP DATABASE IF EXISTS {dbname};') heavydb.sql_execute(f'DROP USER IF EXISTS "{user}";') - - -@pytest.mark.parametrize('kind', ('udf', 'udtf')) -def test_numba_heavydb_llvm_mismatch(heavydb, kind): - heavydb.reset() - - # only run this test on a specific environment - target_info = heavydb.targets['cpu'] - server_llvm_version = target_info.llvm_version - client_llvm_version = llvm.llvm_version_info - - if (server_llvm_version[0], client_llvm_version[0]) != (11, 14): - c_llvm = '.'.join(map(str, client_llvm_version)) - s_llvm = '.'.join(map(str, server_llvm_version)) - msg = (f'Test requires server LLVM 14, got {s_llvm}. And client LLVM ' - f'11, got {c_llvm}') - pytest.skip(msg) - - if kind == 'udf': - @heavydb('int32(int32)') - def add(a): - return a + 1 - else: - @heavydb('int32(TableFunctionManager, Column, OutputColumn)') - def column_copy(mgr, inp, out): - size = len(inp) - mgr.set_output_row_size(size) - for i in range(size): - out[i] = inp[i] - return size - - with pytest.warns(LLVMVersionMismatchWarning): - heavydb.register() diff --git a/rbc/warnings.py b/rbc/warnings.py deleted file mode 100644 index f129a41eb..000000000 --- a/rbc/warnings.py +++ /dev/null @@ -1,7 +0,0 @@ - -class LLVMVersionMismatchWarning(UserWarning): - """ - Raised when Numba and HeavyDB uses different LLVM version which is known to - be incompatible/problematic. - """ - pass From 89f1b2f6bea4a87c8e4b4eb46a615b2eb2c74eea Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Mon, 12 Jun 2023 11:17:17 -0300 Subject: [PATCH 26/26] flake8 --- rbc/heavydb/remoteheavydb.py | 1 - rbc/tests/heavydb/test_heavydb.py | 1 - 2 files changed, 2 deletions(-) diff --git a/rbc/heavydb/remoteheavydb.py b/rbc/heavydb/remoteheavydb.py index 919bc7e47..721944a78 100644 --- a/rbc/heavydb/remoteheavydb.py +++ b/rbc/heavydb/remoteheavydb.py @@ -10,7 +10,6 @@ import configparser import numpy import textwrap -import llvmlite.binding as llvm from collections import defaultdict, namedtuple from rbc.remotejit import RemoteJIT, RemoteCallCapsule from rbc.thrift.utils import resolve_includes diff --git a/rbc/tests/heavydb/test_heavydb.py b/rbc/tests/heavydb/test_heavydb.py index f2248615f..ea331a997 100644 --- a/rbc/tests/heavydb/test_heavydb.py +++ b/rbc/tests/heavydb/test_heavydb.py @@ -1,7 +1,6 @@ import itertools import os -import llvmlite.binding as llvm import numpy as np import pytest