lower-half: resolve loader debug file by build ID

mpi-proxy-split/lower-half/Makefile

@@ -43,7 +43,7 @@ default: lower-half
 # Normally, gcc (or clang) will automatically link with libatomic.so.
 # However, for come MPICXX it's hard to know if they cover it.
 # So this is good as a safety measure.
-lower-half: get-symbol-offset.o copy-stack.o patch-trampoline.o lh-func-ptr.o mem-wrapper.o switch-context.o lower-half.o mmap-fixed-noreplace.o ${DMTCP_DEPS}
+lower-half: get-symbol-offset.o build-id-debug.o copy-stack.o patch-trampoline.o lh-func-ptr.o mem-wrapper.o switch-context.o lower-half.o mmap-fixed-noreplace.o ${DMTCP_DEPS}
 	${MPICXX} -g3 ${TEXTSEG_ADDR_FLAG} ${CXXFLAGS} -o $@ $^ -lpthread -ldl -latomic
 
 install: lower-half

mpi-proxy-split/lower-half/build-id-debug.cpp

@@ -0,0 +1,164 @@
+#include "build-id-debug.h"
+
+#include <elf.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <string>
+#include <vector>
+
+namespace {
+
+size_t align_note_value(size_t value)
+{
+    return (value + 3U) & ~static_cast<size_t>(3U);
+}
+
+bool read_exact_at(int fd, void *buffer, size_t size, off_t offset)
+{
+    unsigned char *cursor = static_cast<unsigned char *>(buffer);
+    size_t remaining = size;
+
+    while (remaining > 0) {
+        ssize_t result = pread(fd, cursor, remaining, offset);
+        if (result <= 0) {
+            return false;
+        }
+        cursor += result;
+        remaining -= static_cast<size_t>(result);
+        offset += result;
+    }
+    return true;
+}
+
+std::string build_id_from_elf(const char *path)
+{
+    int fd = open(path, O_RDONLY);
+    if (fd < 0) {
+        return "";
+    }
+
+    Elf64_Ehdr header;
+    if (!read_exact_at(fd, &header, sizeof(header), 0) ||
+        memcmp(header.e_ident, ELFMAG, SELFMAG) != 0 ||
+        header.e_ident[EI_CLASS] != ELFCLASS64 ||
+        header.e_phentsize != sizeof(Elf64_Phdr)) {
+        close(fd);
+        return "";
+    }
+
+    std::vector<Elf64_Phdr> program_headers(header.e_phnum);
+    if (!read_exact_at(fd,
+                       program_headers.data(),
+                       program_headers.size() * sizeof(Elf64_Phdr),
+                       header.e_phoff)) {
+        close(fd);
+        return "";
+    }
+
+    std::string build_id;
+    for (const Elf64_Phdr &program_header : program_headers) {
+        if (program_header.p_type != PT_NOTE ||
+            program_header.p_filesz < sizeof(Elf64_Nhdr)) {
+            continue;
+        }
+
+        std::vector<unsigned char> notes(program_header.p_filesz);
+        if (!read_exact_at(fd,
+                           notes.data(),
+                           notes.size(),
+                           program_header.p_offset)) {
+            continue;
+        }
+
+        size_t cursor = 0;
+        while (cursor + sizeof(Elf64_Nhdr) <= notes.size()) {
+            Elf64_Nhdr note_header;
+            memcpy(&note_header, notes.data() + cursor, sizeof(note_header));
+            cursor += sizeof(note_header);
+
+            const size_t name_offset = cursor;
+            const size_t aligned_name = align_note_value(note_header.n_namesz);
+            if (aligned_name > notes.size() - cursor) {
+                break;
+            }
+            cursor += aligned_name;
+
+            const size_t description_offset = cursor;
+            const size_t aligned_description =
+                align_note_value(note_header.n_descsz);
+            if (aligned_description > notes.size() - cursor) {
+                break;
+            }
+            cursor += aligned_description;
+
+            const bool gnu_note =
+                note_header.n_namesz >= 3 &&
+                name_offset + note_header.n_namesz <= notes.size() &&
+                memcmp(notes.data() + name_offset, "GNU", 3) == 0;
+
+            if (!gnu_note || note_header.n_type != NT_GNU_BUILD_ID ||
+                description_offset + note_header.n_descsz > notes.size()) {
+                continue;
+            }
+
+            static const char hexadecimal[] = "0123456789abcdef";
+            build_id.reserve(note_header.n_descsz * 2U);
+            for (size_t index = 0; index < note_header.n_descsz; ++index) {
+                const unsigned char value =
+                    notes[description_offset + index];
+                build_id.push_back(hexadecimal[value >> 4]);
+                build_id.push_back(hexadecimal[value & 0x0f]);
+            }
+            break;
+        }
+
+        if (!build_id.empty()) {
+            break;
+        }
+    }
+
+    close(fd);
+    return build_id;
+}
+
+}  // namespace
+
+std::string get_build_id_debug_path(const char *elf_path,
+                                    const char *debug_root)
+{
+    if (elf_path == nullptr || *elf_path == '\0') {
+        return "";
+    }
+
+    char resolved_path[PATH_MAX];
+    if (realpath(elf_path, resolved_path) == nullptr) {
+        return "";
+    }
+
+    const std::string build_id = build_id_from_elf(resolved_path);
+    if (build_id.size() < 3) {
+        return "";
+    }
+
+    if (debug_root == nullptr || *debug_root == '\0') {
+        debug_root = getenv("MANA_DEBUG_ROOT");
+    }
+    if (debug_root == nullptr || *debug_root == '\0') {
+        debug_root = "/usr/lib/debug";
+    }
+
+    std::string result(debug_root);
+    while (result.size() > 1 && result.back() == '/') {
+        result.pop_back();
+    }
+    result += "/.build-id/";
+    result += build_id.substr(0, 2);
+    result += "/";
+    result += build_id.substr(2);
+    result += ".debug";
+    return result;
+}

mpi-proxy-split/lower-half/build-id-debug.h

@@ -0,0 +1,9 @@
+#ifndef MANA_BUILD_ID_DEBUG_H
+#define MANA_BUILD_ID_DEBUG_H
+
+#include <string>
+
+std::string get_build_id_debug_path(const char *elf_path,
+                                    const char *debug_root = nullptr);
+
+#endif

mpi-proxy-split/lower-half/lower-half.cpp

@@ -20,6 +20,7 @@
 #include "patch-trampoline.h"
 #include "lower-half-api.h"
 #include "logging.h"
+#include "build-id-debug.h"
 #include "dmtcp.h"
 #include "dmtcprestartinternal.h"
 #include "procmapsarea.h"
@@ -1423,26 +1424,62 @@ patchAuxv(Elf64_auxv_t *av, unsigned long phnum,
  * @return                The offset of the symbol within the ELF binary. The
  *                        function asserts if the symbol is found.
  */
-off_t get_symbol_offset_with_debug(const char *elf_interpreter, const char *symbol)
+off_t
+get_symbol_offset_with_debug(const char *elf_interpreter, const char *symbol)
 {
   off_t offset = get_symbol_offset(elf_interpreter, symbol);
-  if (!offset) {
-    char buf[256] = "/usr/lib/debug";
-    buf[sizeof(buf)-1] = '\0';
-
-    ssize_t rc = 0;
-    rc = readlink(elf_interpreter, buf + strlen(buf), sizeof(buf) - strlen(buf) - 1);
-    if (rc != -1 && access(buf, F_OK) == 0) {
-      // Debian family (Ubuntu, etc.) use this scheme to store debug symbols.
-      //   http://sourceware.org/gdb/onlinedocs/gdb/Separate-Debug-Files.html
-      fprintf(stderr, "Debug symbols for interpreter in: %s\n", buf);
+  if (offset) {
+    return offset;
+  }
+
+  char conventional_path[PATH_MAX] = "/usr/lib/debug";
+  const size_t prefix_length = strlen(conventional_path);
+  const ssize_t link_length = readlink(
+    elf_interpreter,
+    conventional_path + prefix_length,
+    sizeof(conventional_path) - prefix_length - 1
+  );
+
+  if (link_length >= 0) {
+    conventional_path[prefix_length + link_length] = '\0';
+    if (access(conventional_path, R_OK) == 0) {
+      fprintf(stderr,
+              "Debug symbols for interpreter in: %s\n",
+              conventional_path);
+      offset = get_symbol_offset(conventional_path, symbol);
+      if (offset) {
+        return offset;
+      }
     }
-    offset = get_symbol_offset(buf, symbol); // elf interpreter debug path
+  }
+
+  const std::string build_id_path = get_build_id_debug_path(
+    elf_interpreter,
+    getenv("MANA_DEBUG_ROOT")
+  );
+  if (!build_id_path.empty() && access(build_id_path.c_str(), R_OK) == 0) {
+    fprintf(stderr,
+            "Debug symbols for interpreter in: %s\n",
+            build_id_path.c_str());
+    offset = get_symbol_offset(build_id_path.c_str(), symbol);
+    if (offset) {
+      return offset;
+    }
+  }
+
+  fprintf(stderr,
+          "Unable to find symbol %s in %s or its detached debug files.\n",
+          symbol,
+          elf_interpreter);
+  if (link_length >= 0) {
+    fprintf(stderr, "Tried: %s\n", conventional_path);
+  }
+  if (!build_id_path.empty()) {
+    fprintf(stderr, "Tried: %s\n", build_id_path.c_str());
   }
   assert(offset);
   return offset;
 }
-
 int MPI_MANA_Internal(char *dummy) {
   return 0;
 }