Support reading the AlphaTheta OneLibrary format

.gitignore

@@ -12,3 +12,4 @@ compile_commands.json
 # CMake typical build dirs
 /cmake_build*
 /cmake-build*
+

CMakeLists.txt

@@ -49,6 +49,82 @@ option(SYSTEM_DATE_H "Use system installation of date.h" OFF)
 option(SYSTEM_SQLITE "Use system installation of SQLite" ON)
 option(SYSTEM_SQLITE_MODERN_CPP "Use system installation of sqlite_modern_cpp" OFF)
 
+
+# The processor's own AES instructions are worth two orders of magnitude over
+# software AES when reading an encrypted OneLibrary database, so use them where
+# the compiler can be persuaded to emit them.  Whether the processor running the
+# built library has them is a separate question, decided at run time; settled
+# here is only whether the intrinsics compile, and with which flags.  They are
+# applied to `aes_hardware.cpp` alone, so no other translation unit can pick up
+# an instruction the target may not have.
+include(CheckCXXSourceCompiles)
+
+set(DJINTEROP_AES_X86_SOURCE "
+#if defined(_MSC_VER)
+#include <intrin.h>
+#else
+#include <immintrin.h>
+#endif
+int main()
+{
+    __m128i x = _mm_setzero_si128();
+    x = _mm_aesenc_si128(x, x);
+    x = _mm_aesenclast_si128(x, x);
+    x = _mm_aesdec_si128(x, x);
+    x = _mm_aesdeclast_si128(x, x);
+    return _mm_cvtsi128_si32(x);
+}
+")
+
+set(DJINTEROP_AES_ARM64_SOURCE "
+#include <arm_neon.h>
+int main()
+{
+    uint8x16_t x = vdupq_n_u8(0);
+    x = vaesmcq_u8(vaeseq_u8(x, x));
+    x = vaesimcq_u8(vaesdq_u8(x, x));
+    return vgetq_lane_u8(x, 0);
+}
+")
+
+set(DJINTEROP_AES_INTRINSICS_DEFINE "")
+set(DJINTEROP_AES_INTRINSICS_FLAGS "")
+
+# Try one architecture, unless another has already answered.  Bare comes first:
+# MSVC needs no flags at all, and neither does a compiler whose default target
+# already includes the extension, such as Apple's on arm64.
+macro(djinterop_try_aes arch source flags)
+    if(NOT DJINTEROP_AES_INTRINSICS_DEFINE)
+        check_cxx_source_compiles("${source}" DJINTEROP_AES_${arch}_BARE)
+        if(NOT DJINTEROP_AES_${arch}_BARE)
+            set(CMAKE_REQUIRED_FLAGS "${flags}")
+            check_cxx_source_compiles(
+                "${source}" DJINTEROP_AES_${arch}_FLAGGED)
+            unset(CMAKE_REQUIRED_FLAGS)
+            if(DJINTEROP_AES_${arch}_FLAGGED)
+                set(DJINTEROP_AES_INTRINSICS_FLAGS "${flags}")
+            endif()
+        endif()
+        if(DJINTEROP_AES_${arch}_BARE OR DJINTEROP_AES_${arch}_FLAGGED)
+            set(DJINTEROP_AES_INTRINSICS_DEFINE
+                "DJINTEROP_AES_INTRINSICS_${arch}")
+        endif()
+    endif()
+endmacro()
+
+djinterop_try_aes(X86 "${DJINTEROP_AES_X86_SOURCE}" "-maes -msse2")
+djinterop_try_aes(ARM64 "${DJINTEROP_AES_ARM64_SOURCE}" "-march=armv8-a+crypto")
+
+if(DJINTEROP_AES_INTRINSICS_DEFINE)
+    message(STATUS "Using AES instructions where available at run time")
+    set_source_files_properties(
+        src/djinterop/util/crypto/aes_hardware.cpp PROPERTIES
+        COMPILE_DEFINITIONS "${DJINTEROP_AES_INTRINSICS_DEFINE}"
+        COMPILE_FLAGS "${DJINTEROP_AES_INTRINSICS_FLAGS}")
+else()
+    message(STATUS "No AES instructions for this target; using AES tables")
+endif()
+
 add_library(
     DjInterop
     include/djinterop/album_art.hpp
@@ -81,6 +157,7 @@ add_library(
     include/djinterop/engine/v3/track_table.hpp
     include/djinterop/exceptions.hpp
     include/djinterop/musical_key.hpp
+    include/djinterop/onelibrary/onelibrary.hpp
     include/djinterop/pad_color.hpp
     include/djinterop/performance_data.hpp
     include/djinterop/playlist.hpp
@@ -202,14 +279,46 @@ add_library(
     src/djinterop/impl/playlist_impl.hpp
     src/djinterop/impl/track_impl.cpp
     src/djinterop/impl/track_impl.hpp
+    src/djinterop/onelibrary/loader.hpp
+    src/djinterop/onelibrary/onelibrary.cpp
+    src/djinterop/onelibrary/onelibrary_context.hpp
+    src/djinterop/onelibrary/v1/content_table.cpp
+    src/djinterop/onelibrary/v1/crate_impl.cpp
+    src/djinterop/onelibrary/v1/crate_impl.hpp
+    src/djinterop/onelibrary/v1/database_impl.cpp
+    src/djinterop/onelibrary/v1/database_impl.hpp
+    src/djinterop/onelibrary/v1/library.cpp
+    src/djinterop/onelibrary/v1/playlist_impl.cpp
+    src/djinterop/onelibrary/v1/playlist_impl.hpp
+    src/djinterop/onelibrary/v1/playlist_table.cpp
+    src/djinterop/onelibrary/v1/property_table.cpp
+    src/djinterop/onelibrary/v1/track_conversion.cpp
+    src/djinterop/onelibrary/v1/track_conversion.hpp
+    src/djinterop/onelibrary/v1/track_impl.cpp
+    src/djinterop/onelibrary/v1/track_impl.hpp
     src/djinterop/playlist.cpp
     src/djinterop/track.cpp
     src/djinterop/util/chrono.cpp
     src/djinterop/util/chrono.hpp
+    src/djinterop/util/crypto/aes.cpp
+    src/djinterop/util/crypto/aes.hpp
+    src/djinterop/util/crypto/aes_hardware.cpp
+    src/djinterop/util/crypto/aes_hardware.hpp
+    src/djinterop/util/crypto/encrypted_database.hpp
+    # The one implementation of `encrypted_database.hpp`; a SQLCipher-backed
+    # build would name its own here instead.
+    src/djinterop/util/crypto/encrypted_database_builtin.cpp
+    src/djinterop/util/crypto/sha512.cpp
+    src/djinterop/util/crypto/sha512.hpp
+    src/djinterop/util/crypto/sqlcipher_codec.cpp
+    src/djinterop/util/crypto/sqlcipher_codec.hpp
+    src/djinterop/util/crypto/sqlcipher_wal.cpp
+    src/djinterop/util/crypto/sqlcipher_wal.hpp
     src/djinterop/util/filesystem.cpp
     src/djinterop/util/filesystem.hpp
     src/djinterop/util/random.cpp
     src/djinterop/util/random.hpp
+    src/djinterop/util/sqlite_query.hpp
     src/djinterop/util/sqlite_transaction.hpp
 )
 
@@ -239,6 +348,14 @@ target_include_directories(
         $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
         $<INSTALL_INTERFACE:${DJINTEROP_INSTALL_INCLUDEDIR}>)
 
+# Decrypting the pages of a database is spread over the processors available,
+# which is what the standard threading library is needed for.
+set(THREADS_PREFER_PTHREAD_FLAG ON)
+find_package(Threads REQUIRED)
+target_link_libraries(
+    DjInterop PRIVATE
+    Threads::Threads)
+
 # Always rely on system installation of zlib.
 set(ZLIB_MIN_VERSION 1.2.8)
 find_package(ZLIB ${ZLIB_MIN_VERSION} REQUIRED)
@@ -273,6 +390,7 @@ if(SYSTEM_SQLITE)
     target_link_libraries(
         DjInterop PUBLIC
         ${SQLite3_LIBRARIES})
+    set(DJINTEROP_SQLITE_VERSION "${SQLite3_VERSION}")
 else()
     # Use bundled SQLite amalgamation sources.
     message(STATUS "Using bundled SQLite...")
@@ -285,6 +403,16 @@ else()
     target_include_directories(
         DjInterop PRIVATE SYSTEM
         ext/sqlite-amalgamation)
+
+    # Read the version out of the amalgamation rather than restating it here,
+    # so that bumping the bundled copy is a matter of replacing two files.
+    file(
+        STRINGS ext/sqlite-amalgamation/sqlite3.h DJINTEROP_SQLITE_VERSION
+        REGEX "^#define SQLITE_VERSION[ \t]+\"")
+    string(
+        REGEX REPLACE "^#define SQLITE_VERSION[ \t]+\"([^\"]+)\".*" "\\1"
+        DJINTEROP_SQLITE_VERSION "${DJINTEROP_SQLITE_VERSION}")
+    message(STATUS "Bundled SQLite is version ${DJINTEROP_SQLITE_VERSION}")
 endif()
 
 if(SYSTEM_SQLITE_MODERN_CPP)
@@ -357,6 +485,17 @@ install(FILES
     include/djinterop/engine/v3/track_data_blob.hpp
     include/djinterop/engine/v3/track_table.hpp
     DESTINATION "${DJINTEROP_INSTALL_INCLUDEDIR}/engine/v3")
+install(FILES
+    include/djinterop/onelibrary/onelibrary.hpp
+    DESTINATION "${DJINTEROP_INSTALL_INCLUDEDIR}/onelibrary")
+
+install(
+    FILES
+    include/djinterop/onelibrary/v1/content_table.hpp
+    include/djinterop/onelibrary/v1/library.hpp
+    include/djinterop/onelibrary/v1/playlist_table.hpp
+    include/djinterop/onelibrary/v1/property_table.hpp
+    DESTINATION "${DJINTEROP_INSTALL_INCLUDEDIR}/onelibrary/v1")
 
 
 if (UNIX)
@@ -407,6 +546,7 @@ if (BUILD_EXAMPLES)
 
     add_djinterop_example(engine_prime)
     add_djinterop_example(engine_library_v2_low_level)
+    add_djinterop_example(onelibrary)
 endif()
 
 # Unit tests.
@@ -431,6 +571,10 @@ if (Boost_FOUND)
                 ${Boost_INCLUDE_DIRS}
                 ${CMAKE_CURRENT_BINARY_DIR}/include
                 include)
+        # A test that compiles library sources into itself, rather than only
+        # linking, needs the headers that those sources include.
+        target_include_directories(${test_executable_name} PRIVATE SYSTEM
+                $<TARGET_PROPERTY:DjInterop,INCLUDE_DIRECTORIES>)
         target_link_libraries(${test_executable_name} PUBLIC
                 DjInterop
                 ${Boost_LIBRARIES})
@@ -450,6 +594,64 @@ if (Boost_FOUND)
     add_djinterop_test(engine/v3/ performance_data_table_test)
     add_djinterop_test(engine/v3/ track_table_test)
 
+    # Some parts of the library are internal, and its symbols are hidden, so
+    # tests of them compile those sources into themselves rather than linking.
+    #
+    # The OneLibrary tests are commented out below.  Each of them talks to
+    # SQLite directly, and a test cannot borrow SQLite from the library: its
+    # symbols are hidden there.  With `-DSYSTEM_SQLITE=OFF` that means every
+    # one of them compiles the amalgamation into itself, some fifteen seconds
+    # apiece, on top of the copy the library already builds.
+    #
+    # The sources are still in `test/djinterop/onelibrary`; uncommenting the
+    # block below is all it takes to run them again.  `sqlite3_deserialize`,
+    # which reading a decrypted database goes through, arrived in SQLite 3.36,
+    # hence the version the database test asks for.
+    #
+    # Both problems go away once the format can be written: a fixture would
+    # then be built from `testdata/ref/onelibrary/schema.sql` through the
+    # library, as the Engine tests build one through
+    # `create_database_from_scripts()`, and no test would touch SQLite at all.
+    #
+    # if(SYSTEM_SQLITE)
+    #     set(DJINTEROP_TEST_SQLITE_SOURCES "")
+    # else()
+    #     set(DJINTEROP_TEST_SQLITE_SOURCES ext/sqlite-amalgamation/sqlite3.c)
+    # endif()
+    #
+    # add_djinterop_test(onelibrary/ content_table_test)
+    # target_sources(onelibrary_content_table_test PRIVATE
+    #         src/djinterop/onelibrary/v1/content_table.cpp
+    #         src/djinterop/onelibrary/v1/track_conversion.cpp
+    #         ${DJINTEROP_TEST_SQLITE_SOURCES})
+    #
+    # if(NOT DJINTEROP_SQLITE_VERSION VERSION_LESS 3.36)
+    #     add_djinterop_test(onelibrary/ database_test)
+    #     target_sources(onelibrary_database_test PRIVATE
+    #             src/djinterop/util/crypto/aes.cpp
+    #             src/djinterop/util/crypto/aes_hardware.cpp
+    #             src/djinterop/util/crypto/sha512.cpp
+    #             src/djinterop/util/crypto/sqlcipher_codec.cpp
+    #             src/djinterop/util/filesystem.cpp
+    #             ${DJINTEROP_TEST_SQLITE_SOURCES})
+    # else()
+    #     message(
+    #         STATUS
+    #         "OneLibrary database test not available, as reading the format "
+    #         "needs SQLite 3.36 or newer")
+    # endif()
+    #
+    # add_djinterop_test(onelibrary/ playlist_table_test)
+    # target_sources(onelibrary_playlist_table_test PRIVATE
+    #         src/djinterop/onelibrary/v1/playlist_table.cpp
+    #         ${DJINTEROP_TEST_SQLITE_SOURCES})
+
+    add_djinterop_test(util/ crypto_test)
+    target_sources(util_crypto_test PRIVATE
+            src/djinterop/util/crypto/aes.cpp
+            src/djinterop/util/crypto/aes_hardware.cpp
+            src/djinterop/util/crypto/sha512.cpp
+            src/djinterop/util/crypto/sqlcipher_codec.cpp)
 else()
     message(STATUS "Unit tests not available, as Boost cannot be found")
 endif()

DjInteropConfig.cmake.in

@@ -7,6 +7,8 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}")
 
 include(CMakeFindDependencyMacro)
 find_dependency(ZLIB)
+set(THREADS_PREFER_PTHREAD_FLAG ON)
+find_dependency(Threads)
 if(DJINTEROP_SYSTEM_DATE_H)
   find_dependency(date)
 endif()

GUIDE.md

@@ -73,9 +73,53 @@ As such, in order to create a new library or load an existing library with the
 intention of operating on it using the high-level API, it is always necessary
 to start with format-specific functions to do so:
 
-| Library Type | Include Path                             |
-|--------------|------------------------------------------|
-| Engine       | `#include <djinterop/engine/engine.hpp>` |
+| Library Type | Include Path                                     |
+|--------------|--------------------------------------------------|
+| Engine       | `#include <djinterop/engine/engine.hpp>`         |
+| OneLibrary   | `#include <djinterop/onelibrary/onelibrary.hpp>` |
+
+OneLibrary
+----------
+
+The AlphaTheta OneLibrary format, also documented as Device Library Plus, is
+the successor to the DeviceSQL `export.pdb` library that rekordbox wrote to USB
+media.  A database is loaded by way of `onelibrary::load_database()`, given
+either the root directory of a device or the database file itself.  A number of
+aspects of the format are worth noting:
+
+* Support is currently read-only, and everything that would change a database
+  throws `djinterop::unsupported_operation`.
+* Beat grids, waveforms, hot cues and loops are not held in the database.
+  rekordbox leaves them in the ANLZ files that `content.analysisDataFilePath`
+  points at, and does not populate the `cue` table on export.  Those accessors
+  therefore return nothing rather than throwing.  A caller that reads ANLZ
+  files itself reaches that path through the low-level API, described below.
+* The low-level API also reaches the two other things a device carries that
+  the format-agnostic interface has nowhere to put: the musical key in the
+  notation rekordbox wrote, which may be Camelot and which `track::key()`
+  cannot represent, and the colour the DJ marked a track with, numbered as
+  `export.pdb` numbers them.
+* The format has a single tree that serves as both playlists and crates, so
+  `playlists_and_crates_are_distinct` is false and the two views show the same
+  rows.
+* A device is read by decrypting it into memory.  rekordbox writes the library
+  in write-ahead-logged mode, and the log has to be folded in before SQLite
+  sees the file, so reading one needs SQLite 3.36 or newer, built without
+  `SQLITE_OMIT_DESERIALIZE`.
+
+### OneLibrary low-level API
+
+The low-level API is in `<djinterop/onelibrary/v1/library.hpp>`, and exposes
+the tables as the device holds them, translating no further than resolving a
+lookup reference to the text behind it.  A device is loaded as an
+`onelibrary::v1::library`, whose `content()`, `playlist()` and `property()`
+give the tables, and whose `database()` gives the same database that
+`load_database()` would have.
+
+The `v1` is the schema those tables describe.  A device records it as
+`property.dbVersion`, and every export seen so far reports `1000`.  A schema
+not compatible with this one gets a namespace of its own, as the Engine
+formats do.
 
 
 Stable API/ABI

README.md

@@ -8,7 +8,8 @@ State of Support
 ================
 
 The library is currently in development, and not all features are implemented
-yet.  It currently supports only the Engine Library format.
+yet.  It supports the Engine Library format, and can read the AlphaTheta
+OneLibrary format.
 
 What is supported:
 
@@ -27,11 +28,20 @@ The library supports the following firmware and application versions:
     SC6000/M) may work, but this is currently untested.
 * Engine DJ Desktop (aka Engine Prime) from 1.0.1 to 4.3.0.
 
+The library also reads the AlphaTheta OneLibrary format, which rekordbox 7
+writes to USB media as `PIONEER/rekordbox/exportLibrary.db`, and which players
+from the CDJ-3000X, XDJ-AZ, OPUS-QUAD and OMNIS-DUO onwards read, as does the
+CDJ-3000 from firmware 3.15.  Track metadata, playlists and crates can be read.
+Writing is not supported yet, and beat grids, waveforms, hot cues and loops are
+not held in the database at all, as rekordbox leaves them in the ANLZ files
+beside it.  The format is also documented under the name Device Library Plus.
+
 What is not supported (yet):
 
 * Album art
 * Play history
-* DJ record libraries in formats other than Engine Prime
+* Writing OneLibrary databases
+* DJ record libraries in formats other than Engine Prime and OneLibrary
 
 How Do I Use It?
 ================