diff --git a/mpi-proxy-split/lower-half/Makefile b/mpi-proxy-split/lower-half/Makefile index 2e998d84c..6d25e93d3 100644 --- a/mpi-proxy-split/lower-half/Makefile +++ b/mpi-proxy-split/lower-half/Makefile @@ -43,7 +43,7 @@ default: lower-half # Normally, gcc (or clang) will automatically link with libatomic.so. # However, for come MPICXX it's hard to know if they cover it. # So this is good as a safety measure. -lower-half: get-symbol-offset.o copy-stack.o patch-trampoline.o lh-func-ptr.o mem-wrapper.o switch-context.o lower-half.o mmap-fixed-noreplace.o ${DMTCP_DEPS} +lower-half: get-symbol-offset.o build-id-debug.o copy-stack.o patch-trampoline.o lh-func-ptr.o mem-wrapper.o switch-context.o lower-half.o mmap-fixed-noreplace.o ${DMTCP_DEPS} ${MPICXX} -g3 ${TEXTSEG_ADDR_FLAG} ${CXXFLAGS} -o $@ $^ -lpthread -ldl -latomic install: lower-half diff --git a/mpi-proxy-split/lower-half/build-id-debug.cpp b/mpi-proxy-split/lower-half/build-id-debug.cpp new file mode 100644 index 000000000..e2b5b1632 --- /dev/null +++ b/mpi-proxy-split/lower-half/build-id-debug.cpp @@ -0,0 +1,164 @@ +#include "build-id-debug.h" + +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace { + +size_t align_note_value(size_t value) +{ + return (value + 3U) & ~static_cast(3U); +} + +bool read_exact_at(int fd, void *buffer, size_t size, off_t offset) +{ + unsigned char *cursor = static_cast(buffer); + size_t remaining = size; + + while (remaining > 0) { + ssize_t result = pread(fd, cursor, remaining, offset); + if (result <= 0) { + return false; + } + cursor += result; + remaining -= static_cast(result); + offset += result; + } + return true; +} + +std::string build_id_from_elf(const char *path) +{ + int fd = open(path, O_RDONLY); + if (fd < 0) { + return ""; + } + + Elf64_Ehdr header; + if (!read_exact_at(fd, &header, sizeof(header), 0) || + memcmp(header.e_ident, ELFMAG, SELFMAG) != 0 || + header.e_ident[EI_CLASS] != ELFCLASS64 || + header.e_phentsize != sizeof(Elf64_Phdr)) { + close(fd); + return ""; + } + + std::vector program_headers(header.e_phnum); + if (!read_exact_at(fd, + program_headers.data(), + program_headers.size() * sizeof(Elf64_Phdr), + header.e_phoff)) { + close(fd); + return ""; + } + + std::string build_id; + for (const Elf64_Phdr &program_header : program_headers) { + if (program_header.p_type != PT_NOTE || + program_header.p_filesz < sizeof(Elf64_Nhdr)) { + continue; + } + + std::vector notes(program_header.p_filesz); + if (!read_exact_at(fd, + notes.data(), + notes.size(), + program_header.p_offset)) { + continue; + } + + size_t cursor = 0; + while (cursor + sizeof(Elf64_Nhdr) <= notes.size()) { + Elf64_Nhdr note_header; + memcpy(¬e_header, notes.data() + cursor, sizeof(note_header)); + cursor += sizeof(note_header); + + const size_t name_offset = cursor; + const size_t aligned_name = align_note_value(note_header.n_namesz); + if (aligned_name > notes.size() - cursor) { + break; + } + cursor += aligned_name; + + const size_t description_offset = cursor; + const size_t aligned_description = + align_note_value(note_header.n_descsz); + if (aligned_description > notes.size() - cursor) { + break; + } + cursor += aligned_description; + + const bool gnu_note = + note_header.n_namesz >= 3 && + name_offset + note_header.n_namesz <= notes.size() && + memcmp(notes.data() + name_offset, "GNU", 3) == 0; + + if (!gnu_note || note_header.n_type != NT_GNU_BUILD_ID || + description_offset + note_header.n_descsz > notes.size()) { + continue; + } + + static const char hexadecimal[] = "0123456789abcdef"; + build_id.reserve(note_header.n_descsz * 2U); + for (size_t index = 0; index < note_header.n_descsz; ++index) { + const unsigned char value = + notes[description_offset + index]; + build_id.push_back(hexadecimal[value >> 4]); + build_id.push_back(hexadecimal[value & 0x0f]); + } + break; + } + + if (!build_id.empty()) { + break; + } + } + + close(fd); + return build_id; +} + +} // namespace + +std::string get_build_id_debug_path(const char *elf_path, + const char *debug_root) +{ + if (elf_path == nullptr || *elf_path == '\0') { + return ""; + } + + char resolved_path[PATH_MAX]; + if (realpath(elf_path, resolved_path) == nullptr) { + return ""; + } + + const std::string build_id = build_id_from_elf(resolved_path); + if (build_id.size() < 3) { + return ""; + } + + if (debug_root == nullptr || *debug_root == '\0') { + debug_root = getenv("MANA_DEBUG_ROOT"); + } + if (debug_root == nullptr || *debug_root == '\0') { + debug_root = "/usr/lib/debug"; + } + + std::string result(debug_root); + while (result.size() > 1 && result.back() == '/') { + result.pop_back(); + } + result += "/.build-id/"; + result += build_id.substr(0, 2); + result += "/"; + result += build_id.substr(2); + result += ".debug"; + return result; +} diff --git a/mpi-proxy-split/lower-half/build-id-debug.h b/mpi-proxy-split/lower-half/build-id-debug.h new file mode 100644 index 000000000..9516a4363 --- /dev/null +++ b/mpi-proxy-split/lower-half/build-id-debug.h @@ -0,0 +1,9 @@ +#ifndef MANA_BUILD_ID_DEBUG_H +#define MANA_BUILD_ID_DEBUG_H + +#include + +std::string get_build_id_debug_path(const char *elf_path, + const char *debug_root = nullptr); + +#endif diff --git a/mpi-proxy-split/lower-half/lower-half.cpp b/mpi-proxy-split/lower-half/lower-half.cpp index f2ebe3324..688f657e1 100644 --- a/mpi-proxy-split/lower-half/lower-half.cpp +++ b/mpi-proxy-split/lower-half/lower-half.cpp @@ -20,6 +20,7 @@ #include "patch-trampoline.h" #include "lower-half-api.h" #include "logging.h" +#include "build-id-debug.h" #include "dmtcp.h" #include "dmtcprestartinternal.h" #include "procmapsarea.h" @@ -1423,26 +1424,62 @@ patchAuxv(Elf64_auxv_t *av, unsigned long phnum, * @return The offset of the symbol within the ELF binary. The * function asserts if the symbol is found. */ -off_t get_symbol_offset_with_debug(const char *elf_interpreter, const char *symbol) +off_t +get_symbol_offset_with_debug(const char *elf_interpreter, const char *symbol) { off_t offset = get_symbol_offset(elf_interpreter, symbol); - if (!offset) { - char buf[256] = "/usr/lib/debug"; - buf[sizeof(buf)-1] = '\0'; - - ssize_t rc = 0; - rc = readlink(elf_interpreter, buf + strlen(buf), sizeof(buf) - strlen(buf) - 1); - if (rc != -1 && access(buf, F_OK) == 0) { - // Debian family (Ubuntu, etc.) use this scheme to store debug symbols. - // http://sourceware.org/gdb/onlinedocs/gdb/Separate-Debug-Files.html - fprintf(stderr, "Debug symbols for interpreter in: %s\n", buf); + if (offset) { + return offset; + } + + char conventional_path[PATH_MAX] = "/usr/lib/debug"; + const size_t prefix_length = strlen(conventional_path); + const ssize_t link_length = readlink( + elf_interpreter, + conventional_path + prefix_length, + sizeof(conventional_path) - prefix_length - 1 + ); + + if (link_length >= 0) { + conventional_path[prefix_length + link_length] = '\0'; + if (access(conventional_path, R_OK) == 0) { + fprintf(stderr, + "Debug symbols for interpreter in: %s\n", + conventional_path); + offset = get_symbol_offset(conventional_path, symbol); + if (offset) { + return offset; + } } - offset = get_symbol_offset(buf, symbol); // elf interpreter debug path + } + + const std::string build_id_path = get_build_id_debug_path( + elf_interpreter, + getenv("MANA_DEBUG_ROOT") + ); + if (!build_id_path.empty() && access(build_id_path.c_str(), R_OK) == 0) { + fprintf(stderr, + "Debug symbols for interpreter in: %s\n", + build_id_path.c_str()); + offset = get_symbol_offset(build_id_path.c_str(), symbol); + if (offset) { + return offset; + } + } + + fprintf(stderr, + "Unable to find symbol %s in %s or its detached debug files.\n", + symbol, + elf_interpreter); + if (link_length >= 0) { + fprintf(stderr, "Tried: %s\n", conventional_path); + } + if (!build_id_path.empty()) { + fprintf(stderr, "Tried: %s\n", build_id_path.c_str()); } assert(offset); return offset; } - int MPI_MANA_Internal(char *dummy) { return 0; }