fix(apple): Rewind files after fetching from objectstore

crates/symbolicator-native/src/symbolication/apple.rs

@@ -134,7 +134,7 @@ impl SymbolicationActor {
         sources: Arc<[SourceConfig]>,
         scraping: ScrapingConfig,
     ) -> Result<CompletedSymbolicationResponse> {
-        let report = download_attachment(&self.download_svc, report).await?;
+        let report = download_attachment(self.download_svc.clone(), report).await?;
         let (request, state) =
             self.parse_apple_crash_report(platform, scope, report, sources, scraping)?;
         let mut response = self.symbolicate(request).await?;

crates/symbolicator-native/src/symbolication/attachments.rs

@@ -1,15 +1,14 @@
 use std::fs::File;
-use std::pin::pin;
+use std::sync::Arc;
 
-use futures::TryStreamExt;
-use symbolicator_service::download::DownloadService;
-use tokio::io::{AsyncWriteExt, BufWriter};
-use tokio_util::io::StreamReader;
+use symbolicator_service::download::{DownloadService, fetch_file};
+use symbolicator_sources::{HttpRemoteFile, RemoteFile};
+use url::Url;
 
 use crate::interface::AttachmentFile;
 
 pub async fn download_attachment(
-    download_svc: &DownloadService,
+    download_svc: Arc<DownloadService>,
     file: AttachmentFile,
 ) -> anyhow::Result<File> {
     let (storage_url, storage_token) = match file {
@@ -20,29 +19,20 @@ pub async fn download_attachment(
         } => (storage_url, storage_token),
     };
 
-    // TODO: maybe its worth using the actual `DownloadService` instead of straight going to the `trusted_client`.
-    // Doing so would in theory allow us to have retries and error report, as well as being able to
-    // download files in multiple chunks concurrently, but I don’t think our `objecstore` server currently
-    // supports range requests, and those would also mess with streaming decompression.
-    // Not to mention that using the `DownloadService` is not that straight forward.
-    let mut request = download_svc.trusted_client.get(storage_url);
+    let mut http_remote_file = HttpRemoteFile::from_url(Url::parse(&storage_url)?, true);
+
     if let Some(token) = storage_token {
-        request = request.bearer_auth(token);
+        http_remote_file = http_remote_file.bearer_auth(&token);
     }
-    let stream = request
-        .send()
-        .await?
-        .error_for_status()?
-        .bytes_stream()
-        .map_err(std::io::Error::other);
-    let mut reader = pin!(StreamReader::new(stream));
-
-    let file = tempfile::tempfile()?;
-    let mut writer = BufWriter::new(tokio::fs::File::from_std(file));
-    tokio::io::copy(&mut reader, &mut writer).await?;
-    writer.flush().await?;
-    let file = writer.into_inner();
-    file.sync_data().await?;
-
-    Ok(file.into_std().await)
+
+    let mut temp_file = tempfile::NamedTempFile::new()?;
+
+    fetch_file(
+        download_svc,
+        RemoteFile::Http(http_remote_file),
+        &mut temp_file,
+    )
+    .await?;
+
+    Ok(temp_file.into_file())
 }

crates/symbolicator-native/src/symbolication/process_minidump.rs

@@ -611,7 +611,7 @@ impl SymbolicationActor {
             scraping,
             rewrite_first_module,
         } = request;
-        let minidump_file = download_attachment(&self.download_svc, minidump_file).await?;
+        let minidump_file = download_attachment(self.download_svc.clone(), minidump_file).await?;
         let len = minidump_file.metadata()?.len();
         tracing::debug!("Processing minidump ({} bytes)", len);
         metric!(distribution("minidump.upload.size") = len as f64);

crates/symbolicator-native/tests/integration/main.rs

@@ -1,6 +1,7 @@
 // See <https://matklad.github.io/2021/02/27/delete-cargo-integration-tests.html>
 
 pub mod e2e;
+pub mod process_apple;
 pub mod process_minidump;
 pub mod public_sources;
 pub mod source_errors;

crates/symbolicator-native/tests/integration/process_apple.rs

@@ -0,0 +1,42 @@
+use std::sync::Arc;
+
+use axum::Router;
+use axum::http::header;
+use axum::response::IntoResponse;
+use axum::routing::get;
+use symbolicator_test::Server;
+
+use symbolicator_native::interface::AttachmentFile;
+use symbolicator_service::types::Scope;
+
+use crate::{assert_snapshot, read_fixture, setup_service, symbol_server};
+
+#[tokio::test]
+async fn test_attachment_download() {
+    let (symbolication, _cache_dir) = setup_service(|_| ());
+    let (_symsrv, source) = symbol_server();
+
+    async fn get_crash_report() -> impl IntoResponse {
+        let minidump = read_fixture("apple_crash_report.txt");
+        let compressed = zstd::bulk::compress(&minidump, 0).unwrap();
+
+        ([(header::CONTENT_ENCODING, "zstd")], compressed)
+    }
+    let router = Router::new().route("/the_crash_report.txt", get(get_crash_report));
+    let attachment_server = Server::with_router(router);
+
+    let response = symbolication
+        .process_apple_crash_report(
+            None,
+            Scope::Global,
+            AttachmentFile::Remote {
+                storage_url: attachment_server.url("/the_crash_report.txt").to_string(),
+                storage_token: None,
+            },
+            Arc::new([source]),
+            Default::default(),
+        )
+        .await;
+
+    assert_snapshot!(response.unwrap());
+}

crates/symbolicator-native/tests/integration/snapshots/integration__process_apple__attachment_download.snap

@@ -0,0 +1,215 @@
+---
+source: crates/symbolicator-native/tests/integration/process_apple.rs
+expression: response.unwrap()
+---
+timestamp: 1547055742
+system_info:
+  os_name: macOS
+  os_version: 10.14.0
+  os_build: 18A391
+  cpu_arch: x86_64
+  device_model: "MacBookPro14,3"
+crashed: true
+crash_reason: SIGSEGV
+crash_details: "objc_msgSend() selector name: respondsToSelector:\n  more information here"
+stacktraces:
+  - thread_id: 0
+    is_requesting: false
+    frames:
+      - status: unknown_image
+        original_index: 0
+        instruction_addr: "0x7fff61bc6c2a"
+        package: libsystem_kernel.dylib
+      - status: unknown_image
+        original_index: 1
+        instruction_addr: "0x7fff349f505e"
+        package: CoreFoundation
+      - status: unknown_image
+        original_index: 2
+        instruction_addr: "0x7fff349f45ad"
+        package: CoreFoundation
+      - status: unknown_image
+        original_index: 3
+        instruction_addr: "0x7fff349f3ce4"
+        package: CoreFoundation
+      - status: unknown_image
+        original_index: 4
+        instruction_addr: "0x7fff33c8d895"
+        package: HIToolbox
+      - status: unknown_image
+        original_index: 5
+        instruction_addr: "0x7fff33c8d5cb"
+        package: HIToolbox
+      - status: unknown_image
+        original_index: 6
+        instruction_addr: "0x7fff33c8d348"
+        package: HIToolbox
+      - status: unknown_image
+        original_index: 7
+        instruction_addr: "0x7fff31f4a95b"
+        package: AppKit
+      - status: unknown_image
+        original_index: 8
+        instruction_addr: "0x7fff31f496fa"
+        package: AppKit
+      - status: unknown_image
+        original_index: 9
+        instruction_addr: "0x7fff31f4375d"
+        package: AppKit
+      - status: missing
+        original_index: 10
+        instruction_addr: "0x108b7092b"
+        package: /Users/bruno/Documents/Unreal Projects/YetAnotherMac/MacNoEditor/YetAnotherMac.app/Contents/MacOS/YetAnotherMac
+      - status: missing
+        original_index: 11
+        instruction_addr: "0x108b702a6"
+        package: /Users/bruno/Documents/Unreal Projects/YetAnotherMac/MacNoEditor/YetAnotherMac.app/Contents/MacOS/YetAnotherMac
+      - status: unknown_image
+        original_index: 12
+        instruction_addr: "0x7fff61a8e085"
+        package: libdyld.dylib
+      - status: unknown_image
+        original_index: 13
+        instruction_addr: "0xea004"
+        package: YetanotherMac
+  - thread_id: 1
+    thread_name: Test Thread Name
+    is_requesting: true
+    registers:
+      cs: "0x2b"
+      fs: "0x0"
+      gs: "0x0"
+      r10: "0x0"
+      r11: "0xffffffff"
+      r12: "0x8"
+      r13: "0x11e800b00"
+      r14: "0x1"
+      r15: "0x0"
+      r8: "0x3"
+      r9: "0x10"
+      rax: "0x20261bb4775b008f"
+      rbp: "0x700015a616d0"
+      rbx: "0x0"
+      rcx: "0x1288266c0"
+      rdi: "0x0"
+      rdx: "0x1"
+      rflags: "0x10206"
+      rip: "0x1090a0132"
+      rsi: "0x0"
+      rsp: "0x700015a613f0"
+    frames:
+      - status: unknown_image
+        original_index: 0
+        instruction_addr: "0x7fff61bc85be"
+        package: libsystem_kernel.dylib
+      - status: unknown_image
+        original_index: 1
+        instruction_addr: "0x7fff61c7f415"
+        package: libsystem_pthread.dylib
+      - status: unknown_image
+        original_index: 2
+        instruction_addr: "0x54485244"
+modules:
+  - debug_status: missing
+    features:
+      has_debug_info: false
+      has_unwind_info: false
+      has_symbols: false
+      has_sources: false
+    arch: unknown
+    type: macho
+    code_id: 2d903291397d3d14bfca52c7fb8c5e00
+    code_file: /Users/bruno/Documents/Unreal Projects/YetAnotherMac/MacNoEditor/YetAnotherMac.app/Contents/MacOS/YetAnotherMac
+    debug_id: 2d903291-397d-3d14-bfca-52c7fb8c5e00
+    debug_file: /Users/bruno/Documents/Unreal Projects/YetAnotherMac/MacNoEditor/YetAnotherMac.app/Contents/MacOS/YetAnotherMac
+    image_addr: "0x10864e000"
+    image_size: 108797951
+    candidates:
+      - source: local
+        location: "http://localhost:<port>/symbols/2D90/3291/397D/3D14/BFCA/52C7FB8C5E00"
+        download:
+          status: notfound
+      - source: local
+        location: "http://localhost:<port>/symbols/2D90/3291/397D/3D14/BFCA/52C7FB8C5E00.app"
+        download:
+          status: notfound
+      - source: local
+        location: "http://localhost:<port>/symbols/2D90/3291/397D/3D14/BFCA/52C7FB8C5E00.src.zip"
+        download:
+          status: notfound
+      - source: local
+        location: "http://localhost:<port>/symbols/2d/903291397d3d14bfca52c7fb8c5e00.debug"
+        download:
+          status: notfound
+      - source: local
+        location: "http://localhost:<port>/symbols/YetAnotherMac/2D903291397D3D14BFCA52C7FB8C5E000/YetAnotherMa_"
+        download:
+          status: notfound
+      - source: local
+        location: "http://localhost:<port>/symbols/YetAnotherMac/2D903291397D3D14BFCA52C7FB8C5E000/YetAnotherMac"
+        download:
+          status: notfound
+      - source: local
+        location: "http://localhost:<port>/symbols/YetAnotherMac/2D903291397D3D14BFCA52C7FB8C5E000/YetAnotherMac.src.zip"
+        download:
+          status: notfound
+      - source: local
+        location: "http://localhost:<port>/symbols/YetAnotherMac/2D903291397D3D14BFCA52C7FB8C5E000/YetAnotherMac.sym"
+        download:
+          status: notfound
+  - debug_status: unused
+    features:
+      has_debug_info: false
+      has_unwind_info: false
+      has_symbols: false
+      has_sources: false
+    arch: unknown
+    type: macho
+    code_id: 6deccee4a0523ea4bb67957b06f53ad1
+    code_file: /Users/bruno/Documents/Unreal Projects/YetAnotherMac/MacNoEditor/YetAnotherMac.app/Contents/UE4/Engine/Binaries/ThirdParty/PhysX3/Mac/libPhysX3PROFILE.dylib
+    debug_id: 6deccee4-a052-3ea4-bb67-957b06f53ad1
+    debug_file: /Users/bruno/Documents/Unreal Projects/YetAnotherMac/MacNoEditor/YetAnotherMac.app/Contents/UE4/Engine/Binaries/ThirdParty/PhysX3/Mac/libPhysX3PROFILE.dylib
+    image_addr: "0x112bb2000"
+    image_size: 2170879
+  - debug_status: unused
+    features:
+      has_debug_info: false
+      has_unwind_info: false
+      has_symbols: false
+      has_sources: false
+    arch: unknown
+    type: macho
+    code_id: 5e012a646cc536f19b4da0564049169b
+    code_file: /Users/bruno/Documents/Unreal Projects/YetAnotherMac/MacNoEditor/YetAnotherMac.app/Contents/UE4/Engine/Binaries/ThirdParty/PhysX3/Mac/libPhysX3CookingPROFILE.dylib
+    debug_id: 5e012a64-6cc5-36f1-9b4d-a0564049169b
+    debug_file: /Users/bruno/Documents/Unreal Projects/YetAnotherMac/MacNoEditor/YetAnotherMac.app/Contents/UE4/Engine/Binaries/ThirdParty/PhysX3/Mac/libPhysX3CookingPROFILE.dylib
+    image_addr: "0x112fc0000"
+    image_size: 221183
+  - debug_status: unused
+    features:
+      has_debug_info: false
+      has_unwind_info: false
+      has_symbols: false
+      has_sources: false
+    arch: unknown
+    type: macho
+    code_id: 9c19854471943de6b67e4cc27eed2eab
+    code_file: /Users/bruno/Documents/Unreal Projects/YetAnotherMac/MacNoEditor/YetAnotherMac.app/Contents/UE4/Engine/Binaries/ThirdParty/PhysX3/Mac/libPhysX3CommonPROFILE.dylib
+    debug_id: 9c198544-7194-3de6-b67e-4cc27eed2eab
+    debug_file: /Users/bruno/Documents/Unreal Projects/YetAnotherMac/MacNoEditor/YetAnotherMac.app/Contents/UE4/Engine/Binaries/ThirdParty/PhysX3/Mac/libPhysX3CommonPROFILE.dylib
+    image_addr: "0x113013000"
+    image_size: 1474559
+  - debug_status: unused
+    features:
+      has_debug_info: false
+      has_unwind_info: false
+      has_symbols: false
+      has_sources: false
+    arch: unknown
+    type: macho
+    code_id: 890f0997f90435449af7cf011f09a06e
+    code_file: /Users/bruno/Documents/Unreal Projects/YetAnotherMac/MacNoEditor/YetAnotherMac.app/Contents/UE4/Engine/Binaries/ThirdParty/PhysX3/Mac/libPxFoundationPROFILE.dylib
+    debug_id: 890f0997-f904-3544-9af7-cf011f09a06e
+    debug_file: /Users/bruno/Documents/Unreal Projects/YetAnotherMac/MacNoEditor/YetAnotherMac.app/Contents/UE4/Engine/Binaries/ThirdParty/PhysX3/Mac/libPxFoundationPROFILE.dylib
+    image_addr: "0x1131fa000"
+    image_size: 28671

crates/symbolicator-sources/src/sources/http.rs

@@ -88,6 +88,14 @@ impl HttpRemoteFile {
         HttpRemoteFile::new(source, location)
     }
 
+    /// Adds bearer authorization to the request and returns the updated file.
+    pub fn bearer_auth(mut self, token: &str) -> Self {
+        self.headers
+            .0
+            .insert("Authorization".to_owned(), format!("Bearer {token}"));
+        self
+    }
+
     /// Returns a [`RemoteFileUri`] for the file.
     pub fn uri(&self) -> RemoteFileUri {
         match self.url() {