Fix data export system

app/jobs/user/data_export_job.rb

@@ -24,11 +24,19 @@ def perform(data_export_id)
       end
 
       # Upload synchronously before entering a transaction.
-      blob = ActiveStorage::Blob.create_and_upload!(
-        io: File.open(temp_zip.path),
-        filename: zip_filename,
-        content_type: "application/zip"
-      )
+      blob = File.open(temp_zip.path) do |file|
+        # R2 uploads of export archives use a separate service with a raised multipart threshold
+        blob_options = {
+          io: file,
+          filename: zip_filename,
+          content_type: "application/zip"
+        }
+        blob_options[:service_name] = "cloudflare_exports" if Rails.env.production?
+
+        ActiveStorage::Blob.create_and_upload!(
+          **blob_options
+        )
+      end
 
       @data_export.with_lock do
         # The user may delete an export while it is being generated.
@@ -122,7 +130,11 @@ def download_attachment(zip, attachable, entry_name)
     return unless attachment&.blob
 
     zip.put_next_entry(entry_name)
-    attachment.open { |file| IO.copy_stream(file, zip) }
+    bytes_written = attachment.open { |file| IO.copy_stream(file, zip) }
+    expected_bytes = attachment.blob.byte_size
+    return if bytes_written == expected_bytes
+
+    raise IOError, "Incomplete attachment export for #{attachment.blob.filename}: expected #{expected_bytes} bytes, wrote #{bytes_written}"
   end
 
   def write_readme(zip, user)

config/storage.yml

@@ -17,6 +17,20 @@ cloudflare:
   force_path_style: true
   request_checksum_calculation: "when_required"
   response_checksum_validation: "when_required"
+
+# Data exports on a separate service so their S3 multipart threshold can be raised without changing other R2 uploads.
+cloudflare_exports:
+  service: S3
+  endpoint: https://<%= Rails.application.credentials.dig(:cloudflare, :account_id) %>.r2.cloudflarestorage.com
+  access_key_id: <%= Rails.application.credentials.dig(:cloudflare, :access_key_id) %>
+  secret_access_key: <%= Rails.application.credentials.dig(:cloudflare, :secret_access_key) %>
+  region: auto
+  bucket: <%= Rails.application.credentials.dig(:cloudflare, :bucket) %>
+  force_path_style: true
+  request_checksum_calculation: "when_required"
+  response_checksum_validation: "when_required"
+  upload:
+    multipart_threshold: <%= 5.gigabytes %>
 # Use bin/rails credentials:edit to set the AWS secrets (as aws:access_key_id|secret_access_key)
 # amazon:
 #   service: S3

test/jobs/user/data_export_job_test.rb

@@ -108,6 +108,20 @@ class User::DataExportJobTest < ActiveJob::TestCase
     refute @export.zip_file.attached?
   end
 
+  test "marks export as failed when an attachment stream is truncated" do
+    blob = Struct.new(:byte_size, :filename).new(10, "test.png")
+    attachment = Struct.new(:blob) do
+      def open
+        yield StringIO.new("short")
+      end
+    end.new(blob)
+    zip = Zip::OutputStream.new(StringIO.new, stream: true)
+
+    assert_raises(IOError) do
+      User::DataExportJob.new.send(:download_attachment, zip, attachment, "truncated")
+    end
+  end
+
   test "does nothing when the user deleted the queued export" do
     export_id = @export.id
     @export.destroy!