From b90ac46b2df7ff52b402340df7ac145ae8c62f6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tor=20Arne=20Vestb=C3=B8?= Date: Tue, 28 Apr 2026 02:32:18 +0200 Subject: [PATCH 1/7] Prepare for network modes that do not provide pre-allocated IP addresses - Attachment.ipv4Address and .ipv4Gateway made optional; bridge attachments carry no pre-allocated IP (assigned via DHCP at runtime) - Update all call sites and tests for optional IP fields Co-authored-by: Curd Becker --- Sources/APIServer/ContainerDNSHandler.swift | 6 ++-- .../Builder/BuilderStatus.swift | 2 +- .../Container/ContainerList.swift | 2 +- .../Network/Attachment.swift | 30 ++++++++-------- .../Server/DefaultNetworkService.swift | 4 +-- .../Server/IsolatedInterfaceStrategy.swift | 8 +++-- .../Server/NonisolatedInterfaceStrategy.swift | 5 ++- .../RuntimeLinux/Server/RuntimeService.swift | 8 +++-- .../Subcommands/Networks/TestCLINetwork.swift | 4 +-- .../Subcommands/Run/TestCLIRunCommand.swift | 4 +-- .../AttachmentTest.swift | 34 +++++++++++++++++++ 11 files changed, 75 insertions(+), 32 deletions(-) create mode 100644 Tests/ContainerResourceTests/AttachmentTest.swift diff --git a/Sources/APIServer/ContainerDNSHandler.swift b/Sources/APIServer/ContainerDNSHandler.swift index 78a207467..dd0996283 100644 --- a/Sources/APIServer/ContainerDNSHandler.swift +++ b/Sources/APIServer/ContainerDNSHandler.swift @@ -76,10 +76,12 @@ struct ContainerDNSHandler: DNSHandler { } private func answerHost(question: Question) async throws -> ResourceRecord? { - guard let ipAllocation = try await networkService.lookup(hostname: question.name) else { + guard let ipAllocation = try await networkService.lookup(hostname: question.name), + let ipv4Address = ipAllocation.ipv4Address + else { return nil } - let ipv4 = ipAllocation.ipv4Address.address.description + let ipv4 = ipv4Address.address.description guard let ip = try? IPv4Address(ipv4) else { throw DNSResolverError.serverError("failed to parse IP address: \(ipv4)") } diff --git a/Sources/ContainerCommands/Builder/BuilderStatus.swift b/Sources/ContainerCommands/Builder/BuilderStatus.swift index 3512fa5e0..ac2bb67c7 100644 --- a/Sources/ContainerCommands/Builder/BuilderStatus.swift +++ b/Sources/ContainerCommands/Builder/BuilderStatus.swift @@ -85,7 +85,7 @@ private struct PrintableBuilder: ListDisplayable { snapshot.id, snapshot.configuration.image.reference, snapshot.status.rawValue, - snapshot.networks.map { $0.ipv4Address.description }.joined(separator: ","), + snapshot.networks.map { $0.ipv4Address?.description ?? "" }.joined(separator: ","), "\(snapshot.configuration.resources.cpus)", "\(snapshot.configuration.resources.memoryInBytes / (1024 * 1024)) MB", ] diff --git a/Sources/ContainerCommands/Container/ContainerList.swift b/Sources/ContainerCommands/Container/ContainerList.swift index 39121f465..8a56ae9c1 100644 --- a/Sources/ContainerCommands/Container/ContainerList.swift +++ b/Sources/ContainerCommands/Container/ContainerList.swift @@ -64,7 +64,7 @@ extension PrintableContainer: ListDisplayable { self.configuration.platform.os, self.configuration.platform.architecture, self.status.rawValue, - self.networks.map { $0.ipv4Address.description }.joined(separator: ","), + self.networks.map { $0.ipv4Address?.description ?? "" }.joined(separator: ","), "\(self.configuration.resources.cpus)", "\(self.configuration.resources.memoryInBytes / (1024 * 1024)) MB", self.startedDate?.ISO8601Format() ?? "", diff --git a/Sources/ContainerResource/Network/Attachment.swift b/Sources/ContainerResource/Network/Attachment.swift index bbadad759..bce860544 100644 --- a/Sources/ContainerResource/Network/Attachment.swift +++ b/Sources/ContainerResource/Network/Attachment.swift @@ -23,9 +23,11 @@ public struct Attachment: Codable, Sendable { /// The hostname associated with the attachment. public let hostname: String /// The CIDR address describing the interface IPv4 address, with the prefix length of the subnet. - public let ipv4Address: CIDRv4 + /// Nil for bridge-mode attachments where the address is assigned by DHCP at runtime. + public let ipv4Address: CIDRv4? /// The IPv4 gateway address. - public let ipv4Gateway: IPv4Address + /// Nil for bridge-mode attachments where the gateway is discovered via DHCP at runtime. + public let ipv4Gateway: IPv4Address? /// The CIDR address describing the interface IPv6 address, with the prefix length of the subnet. /// The address is nil if the IPv6 subnet could not be determined at network creation time. public let ipv6Address: CIDRv6? @@ -37,8 +39,8 @@ public struct Attachment: Codable, Sendable { public init( network: String, hostname: String, - ipv4Address: CIDRv4, - ipv4Gateway: IPv4Address, + ipv4Address: CIDRv4?, + ipv4Gateway: IPv4Address?, ipv6Address: CIDRv6?, macAddress: MACAddress?, mtu: UInt32? = nil @@ -72,16 +74,12 @@ public struct Attachment: Codable, Sendable { network = try container.decode(String.self, forKey: .network) hostname = try container.decode(String.self, forKey: .hostname) - if let address = try? container.decode(CIDRv4.self, forKey: .ipv4Address) { - ipv4Address = address - } else { - ipv4Address = try container.decode(CIDRv4.self, forKey: .address) - } - if let gateway = try? container.decode(IPv4Address.self, forKey: .ipv4Gateway) { - ipv4Gateway = gateway - } else { - ipv4Gateway = try container.decode(IPv4Address.self, forKey: .gateway) - } + ipv4Address = + try container.decodeIfPresent(CIDRv4.self, forKey: .ipv4Address) + ?? container.decodeIfPresent(CIDRv4.self, forKey: .address) + ipv4Gateway = + try container.decodeIfPresent(IPv4Address.self, forKey: .ipv4Gateway) + ?? container.decodeIfPresent(IPv4Address.self, forKey: .gateway) ipv6Address = try container.decodeIfPresent(CIDRv6.self, forKey: .ipv6Address) macAddress = try container.decodeIfPresent(MACAddress.self, forKey: .macAddress) mtu = try container.decodeIfPresent(UInt32.self, forKey: .mtu) @@ -93,8 +91,8 @@ public struct Attachment: Codable, Sendable { try container.encode(network, forKey: .network) try container.encode(hostname, forKey: .hostname) - try container.encode(ipv4Address, forKey: .ipv4Address) - try container.encode(ipv4Gateway, forKey: .ipv4Gateway) + try container.encodeIfPresent(ipv4Address, forKey: .ipv4Address) + try container.encodeIfPresent(ipv4Gateway, forKey: .ipv4Gateway) try container.encodeIfPresent(ipv6Address, forKey: .ipv6Address) try container.encodeIfPresent(macAddress, forKey: .macAddress) try container.encodeIfPresent(mtu, forKey: .mtu) diff --git a/Sources/Services/Network/Server/DefaultNetworkService.swift b/Sources/Services/Network/Server/DefaultNetworkService.swift index 29f3fbe82..efbfc883b 100644 --- a/Sources/Services/Network/Server/DefaultNetworkService.swift +++ b/Sources/Services/Network/Server/DefaultNetworkService.swift @@ -83,8 +83,8 @@ public actor DefaultNetworkService: NetworkService { "allocated attachment", metadata: [ "hostname": "\(hostname)", - "ipv4Address": "\(attachment.ipv4Address)", - "ipv4Gateway": "\(attachment.ipv4Gateway)", + "ipv4Address": "\(attachment.ipv4Address?.description ?? "none")", + "ipv4Gateway": "\(attachment.ipv4Gateway?.description ?? "none")", "ipv6Address": "\(attachment.ipv6Address?.description ?? "unavailable")", "macAddress": "\(attachment.macAddress?.description ?? "unspecified")", ]) diff --git a/Sources/Services/RuntimeLinux/Server/IsolatedInterfaceStrategy.swift b/Sources/Services/RuntimeLinux/Server/IsolatedInterfaceStrategy.swift index d180fac5c..ac975b48e 100644 --- a/Sources/Services/RuntimeLinux/Server/IsolatedInterfaceStrategy.swift +++ b/Sources/Services/RuntimeLinux/Server/IsolatedInterfaceStrategy.swift @@ -18,6 +18,7 @@ import ContainerResource import ContainerRuntimeClient import ContainerXPC import Containerization +import ContainerizationError /// Isolated container network interface strategy. This strategy prohibits /// container to container networking, but it is the only approach that @@ -25,10 +26,13 @@ import Containerization public struct IsolatedInterfaceStrategy: InterfaceStrategy { public init() {} - public func toInterface(attachment: Attachment, interfaceIndex: Int, additionalData: XPCMessage?) -> Interface { + public func toInterface(attachment: Attachment, interfaceIndex: Int, additionalData: XPCMessage?) throws -> Interface { + guard let ipv4Address = attachment.ipv4Address else { + throw ContainerizationError(.invalidState, message: "NAT attachment missing IPv4 address") + } let ipv4Gateway = interfaceIndex == 0 ? attachment.ipv4Gateway : nil return NATInterface( - ipv4Address: attachment.ipv4Address, + ipv4Address: ipv4Address, ipv4Gateway: ipv4Gateway, macAddress: attachment.macAddress, // https://github.com/apple/containerization/pull/38 diff --git a/Sources/Services/RuntimeLinux/Server/NonisolatedInterfaceStrategy.swift b/Sources/Services/RuntimeLinux/Server/NonisolatedInterfaceStrategy.swift index 38c1b6764..ea1602d65 100644 --- a/Sources/Services/RuntimeLinux/Server/NonisolatedInterfaceStrategy.swift +++ b/Sources/Services/RuntimeLinux/Server/NonisolatedInterfaceStrategy.swift @@ -42,10 +42,13 @@ public struct NonisolatedInterfaceStrategy: InterfaceStrategy { throw ContainerizationError(.invalidState, message: "cannot deserialize custom network reference, status \(status)") } + guard let ipv4Address = attachment.ipv4Address else { + throw ContainerizationError(.invalidState, message: "NAT attachment missing IPv4 address") + } log.info("creating NATNetworkInterface with network reference") let ipv4Gateway = interfaceIndex == 0 ? attachment.ipv4Gateway : nil return NATNetworkInterface( - ipv4Address: attachment.ipv4Address, + ipv4Address: ipv4Address, ipv4Gateway: ipv4Gateway, reference: networkRef, macAddress: attachment.macAddress, diff --git a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift index 1f23e0966..69d4a72b5 100644 --- a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift +++ b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift @@ -261,8 +261,7 @@ public actor RuntimeService { // NOTE: We can support a user providing new entries eventually, but for now craft // a default /etc/hosts. var hostsEntries = [Hosts.Entry.localHostIPV4()] - if !interfaces.isEmpty { - let primaryIfaceAddr = interfaces[0].ipv4Address + if !interfaces.isEmpty, let primaryIfaceAddr = interfaces[0].ipv4Address { hostsEntries.append( Hosts.Entry( ipAddress: primaryIfaceAddr.address.description, @@ -877,7 +876,10 @@ public actor RuntimeService { let containerIPAddress: String switch publishedPort.hostAddress { case .v4(_): - containerIPAddress = attachment.ipv4Address.address.description + guard let ipv4Address = attachment.ipv4Address else { + throw ContainerizationError(.invalidState, message: "cannot configure IPv4 port forwarding for container with unknown IPv4 address") + } + containerIPAddress = ipv4Address.address.description case .v6(_): guard let ipv6Address = attachment.ipv6Address else { throw ContainerizationError(.invalidState, message: "cannot configure IPv6 port forwarding for container with unknown IPv6 address") diff --git a/Tests/CLITests/Subcommands/Networks/TestCLINetwork.swift b/Tests/CLITests/Subcommands/Networks/TestCLINetwork.swift index 2e30c74a9..f81df9e2f 100644 --- a/Tests/CLITests/Subcommands/Networks/TestCLINetwork.swift +++ b/Tests/CLITests/Subcommands/Networks/TestCLINetwork.swift @@ -72,7 +72,7 @@ class TestCLINetwork: CLITest { let container = try inspectContainer(name) #expect(container.networks.count > 0) - let cidrAddress = container.networks[0].ipv4Address + let cidrAddress = try #require(container.networks[0].ipv4Address) let url = "http://\(cidrAddress.address):\(port)" var request = HTTPClientRequest(url: url) request.method = .GET @@ -244,7 +244,7 @@ class TestCLINetwork: CLITest { let container = try inspectContainer(name) #expect(container.networks.count > 0) let curlImage = "docker.io/curlimages/curl:8.6.0" - let cidrAddress = container.networks[0].ipv4Address + let cidrAddress = try #require(container.networks[0].ipv4Address) let url = "http://\(cidrAddress.address):\(port)" let (_, _, _, succeed) = try run(arguments: [ "run", diff --git a/Tests/CLITests/Subcommands/Run/TestCLIRunCommand.swift b/Tests/CLITests/Subcommands/Run/TestCLIRunCommand.swift index 4d7ec61bd..2b9e12f3b 100644 --- a/Tests/CLITests/Subcommands/Run/TestCLIRunCommand.swift +++ b/Tests/CLITests/Subcommands/Run/TestCLIRunCommand.swift @@ -610,7 +610,7 @@ class TestCLIRunCommand3: CLITest { .map { $0.joined(separator: " ") } let inspectOutput = try inspectContainer(name) - let ip = inspectOutput.networks[0].ipv4Address.address + let ip = try #require(inspectOutput.networks[0].ipv4Address).address let expectedNameserver = IPv4Address((ip.value & Prefix(length: 24)!.prefixMask32) + 1).description let defaultDomain = try getDefaultDomain() let expectedLines: [String] = [ @@ -672,7 +672,7 @@ class TestCLIRunCommand3: CLITest { } let inspectOutput = try inspectContainer(name) - let ip = inspectOutput.networks[0].ipv4Address.address + let ip = try #require(inspectOutput.networks[0].ipv4Address).address let output = try doExec(name: name, cmd: ["cat", "/etc/hosts"]) let lines = output.split(separator: "\n") diff --git a/Tests/ContainerResourceTests/AttachmentTest.swift b/Tests/ContainerResourceTests/AttachmentTest.swift new file mode 100644 index 000000000..60b7e50ef --- /dev/null +++ b/Tests/ContainerResourceTests/AttachmentTest.swift @@ -0,0 +1,34 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Testing + +@testable import ContainerResource + +struct AttachmentTest { + @Test func testAttachmentNilIPFields() { + let attachment = Attachment( + network: "my-net", + hostname: "host1", + ipv4Address: nil, + ipv4Gateway: nil, + ipv6Address: nil, + macAddress: nil + ) + #expect(attachment.ipv4Address == nil) + #expect(attachment.ipv4Gateway == nil) + } +} From 0801fd57aef63d3cf582a6e5460f9bdafafdd48a Mon Sep 17 00:00:00 2001 From: Curd Becker Date: Mon, 1 Jun 2026 01:53:01 +0200 Subject: [PATCH 2/7] Add missing getter to retrieve XPCDictionary from an XPCMessage The setter for adding an XPCDictionary to an XPCMessage is present, but the getter to retrieve it again is missing. This might be connected to the fact that xpc_endpoint_t is a type alias to xpc_object_t, so they appear to use an identical setter implementation (unless there is some implicit argument overloading that I am not aware of), but they apparently they cannot use the same getter method, since there is a dedicated `xpc_dictionary_get_dictionary` method to retrieve the dictionary again. --- Sources/ContainerXPC/XPCMessage.swift | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Sources/ContainerXPC/XPCMessage.swift b/Sources/ContainerXPC/XPCMessage.swift index 3c6a3dca8..d5f3ba385 100644 --- a/Sources/ContainerXPC/XPCMessage.swift +++ b/Sources/ContainerXPC/XPCMessage.swift @@ -275,6 +275,12 @@ extension XPCMessage { } } + public func xpcDictionary(key: String) -> xpc_object_t? { + lock.withLock { + xpc_dictionary_get_dictionary(self.object, key) + } + } + public func endpoint(key: String) -> xpc_endpoint_t? { lock.withLock { xpc_dictionary_get_value(self.object, key) From cee18475a28d055e44c0c651d3de8cf8c4a64e9a Mon Sep 17 00:00:00 2001 From: Curd Becker Date: Mon, 1 Jun 2026 01:58:02 +0200 Subject: [PATCH 3/7] Pass network configuration options along to vmnet helper Right now, the configuration for a network is only saved in the `NetworksService` after the network has been successfully started. However, that constitutes a dependency issue when the vmnet helper implementation depends on options in order to be able to create and start the network in the first place. This commit fixes this issue by passing the options (excluding the variant) explicitly along as a serialized json dictionary. This is not that elegant, but works well for now. To me it looks also that this area is still undergoing quite some refactoring, so this might be a workaround until a more elegant permanent solution is implace. --- .../NetworkVmnet/NetworkVmnetHelper+Start.swift | 13 ++++++++++++- .../Server/Networks/NetworksService.swift | 11 +++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift b/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift index 994933489..666832a67 100644 --- a/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift +++ b/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift @@ -67,10 +67,19 @@ extension NetworkVmnetHelper { return .reserved }() + @Option(name: .customLong("option-json"), help: "UTF8-encoded JSON string that contains the configuration options for this network") + var stringifiedOptions: String = "{}" + var logRoot = LogRoot.path func run() async throws { let commandName = NetworkVmnetHelper._commandName + guard let encodedOptions = stringifiedOptions.data(using: .utf8), + let options = try? JSONSerialization.jsonObject(with: encodedOptions) as? [String: String] + else { + throw ContainerizationError(.invalidArgument, message: "failed to decode network configuration options from JSON string") + } + let logPath = logRoot.map { $0.appending("\(commandName)-\(id).log") } let log = ServiceLogger.bootstrap(category: "NetworkVmnetHelper", metadata: ["id": "\(id)"], debug: debug, logPath: logPath) log.info("starting helper", metadata: ["name": "\(commandName)"]) @@ -89,7 +98,9 @@ extension NetworkVmnetHelper { ipv4Subnet: ipv4Subnet, ipv6Subnet: ipv6Subnet, plugin: NetworkVmnetHelper._commandName, - options: ["variant": self.variant.rawValue] + options: options.merging( + ["variant": self.variant.rawValue], + uniquingKeysWith: { _, new in new }) ) let network = try Self.createNetwork( configuration: configuration, diff --git a/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift b/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift index 92655d229..20f8de2d9 100644 --- a/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift +++ b/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift @@ -428,6 +428,17 @@ public actor NetworksService { args += ["--variant", variant] } + // TODO: variant could possibly stay inside the options and does not need to be a dedicated commandline argument? + let options = configuration.options.filter({ key, _ in key != "variant" }) + if !options.isEmpty { + guard let encodedOptions = try? JSONSerialization.data(withJSONObject: options), + let stringifiedOptions = String(data: encodedOptions, encoding: .utf8) + else { + throw ContainerizationError(.internalError, message: "failed to encode network configuration options as json string") + } + args += ["--option-json", stringifiedOptions] + } + let entityPath = try store.entityPath(configuration.id) try pluginLoader.registerWithLaunchd( plugin: networkPlugin, From 206ac5cebd470b395a038bb7add8b4184a58de0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tor=20Arne=20Vestb=C3=B8?= Date: Tue, 28 Apr 2026 16:47:38 +0200 Subject: [PATCH 4/7] Add kernel IP_PNP cmdline args for bridge networking For each bridge attachment (ipv4Address == nil) at interface index n, append ip=:::::eth:dhcp to the kernel cmdline before booting the VM. The kernel's built-in DHCP client (CONFIG_IP_PNP_DHCP=y) acquires the lease before userspace starts, so no DHCP binary is needed in the rootfs. We also pass the hostname to the DHCP server. VZVirtualMachineManager creation is moved after attachment allocation and interface creation, so interface indices are known when the cmdline is finalized. Co-authored-by: Curd Becker --- .../RuntimeLinux/Server/RuntimeService.swift | 39 ++++++++++++------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift index 69d4a72b5..3fc084821 100644 --- a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift +++ b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift @@ -162,12 +162,6 @@ public actor RuntimeService { var kernel = try bundle.kernel kernel.commandLine.kernelArgs.append("oops=panic") kernel.commandLine.kernelArgs.append("lsm=lockdown,capability,landlock,yama,apparmor") - let vmm = VZVirtualMachineManager( - kernel: kernel, - initialFilesystem: bundle.initialFilesystem.asMount, - rosetta: config.rosetta, - logger: self.log - ) let networkBootstrapInfos = try message.networkBootstrapInfos() @@ -196,6 +190,12 @@ public actor RuntimeService { mtu: mtu ) } + + // enable DHCP if the attachment has not been assigned an explicit IP address + if attachment.ipv4Address == nil { + kernel.commandLine.kernelArgs.append("ip=::::\(attachment.hostname):eth\(index):dhcp") + } + guard let iStrategy = self.interfaceStrategies[NetworkInterfaceKey(plugin: info.plugin, variant: info.options["variant"])] else { throw ContainerizationError( .internalError, @@ -214,17 +214,23 @@ public actor RuntimeService { throw error } + let vmm = VZVirtualMachineManager( + kernel: kernel, + initialFilesystem: bundle.initialFilesystem.asMount, + rosetta: config.rosetta, + logger: self.log + ) + // Dynamically configure the DNS nameserver from a network if no explicit configuration + // For bridge networks (unspecified gateway), nameservers and domain come from DHCP (/proc/net/pnp). if let dns = config.dns, dns.nameservers.isEmpty { let defaultNameservers = self.getDefaultNameservers(from: attachments) - if !defaultNameservers.isEmpty { - config.dns = ContainerConfiguration.DNSConfiguration( - nameservers: defaultNameservers, - domain: dns.domain, - searchDomains: dns.searchDomains, - options: dns.options - ) - } + config.dns = ContainerConfiguration.DNSConfiguration( + nameservers: defaultNameservers.isEmpty ? dns.nameservers : defaultNameservers, + domain: defaultNameservers.isEmpty ? nil : dns.domain, + searchDomains: dns.searchDomains, + options: dns.options + ) } let stdio = message.stdio() @@ -1042,7 +1048,10 @@ public actor RuntimeService { private nonisolated func getDefaultNameservers(from attachments: [Attachment]) -> [String] { for attachment in attachments { - return [attachment.ipv4Gateway.description] + guard let ipv4Gateway: IPv4Address = attachment.ipv4Gateway else { + continue + } + return [ipv4Gateway.description] } return [] } From b5f2c06a10a6475ed82c6839119e2b40e77f4a5d Mon Sep 17 00:00:00 2001 From: Curd Becker Date: Mon, 1 Jun 2026 14:01:47 +0200 Subject: [PATCH 5/7] Expose NetworkVariant as global enum in ContainerNetworkClient This was previously just a private enum inside the NetworkVmnetHelper entrypoint, but it makes sense to make this actually globally accessible ... at least for now. I imagine this is also going to be likely the focus of the ongoing refactoring. --- .../NetworkVmnetHelper+Start.swift | 10 +++------- .../Network/Client/NetworkVariant.swift | 20 +++++++++++++++++++ 2 files changed, 23 insertions(+), 7 deletions(-) create mode 100644 Sources/Services/Network/Client/NetworkVariant.swift diff --git a/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift b/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift index 666832a67..08005f6ba 100644 --- a/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift +++ b/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift @@ -27,12 +27,8 @@ import ContainerizationExtras import Foundation import Logging -enum Variant: String, ExpressibleByArgument { - case reserved - case allocationOnly -} - extension NetworkMode: ExpressibleByArgument {} +extension NetworkVariant: ExpressibleByArgument {} extension NetworkVmnetHelper { struct Start: AsyncParsableCommand { @@ -60,7 +56,7 @@ extension NetworkVmnetHelper { var ipv6Subnet: String? @Option(name: .long, help: "Variant of the network helper to use.") - var variant: Variant = { + var variant: NetworkVariant = { guard #available(macOS 26, *) else { return .allocationOnly } @@ -133,7 +129,7 @@ extension NetworkVmnetHelper { } } - private static func createNetwork(configuration: NetworkConfiguration, variant: Variant, log: Logger) throws -> Network { + private static func createNetwork(configuration: NetworkConfiguration, variant: NetworkVariant, log: Logger) throws -> Network { switch variant { case .allocationOnly: return try AllocationOnlyVmnetNetwork(configuration: configuration, log: log) diff --git a/Sources/Services/Network/Client/NetworkVariant.swift b/Sources/Services/Network/Client/NetworkVariant.swift new file mode 100644 index 000000000..99a30b7b3 --- /dev/null +++ b/Sources/Services/Network/Client/NetworkVariant.swift @@ -0,0 +1,20 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +public enum NetworkVariant: String, Sendable { + case reserved + case allocationOnly +} From 9f64129362da1f19588c3f0aa0605b722d77d6eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tor=20Arne=20Vestb=C3=B8?= Date: Tue, 28 Apr 2026 17:06:52 +0200 Subject: [PATCH 6/7] Add bridged vmnet network plugin variant including BridgedInterfaceStrategy Implements the vmnet plugin side of bridge networking: a new BridgedVmnetNetwork actor that uses placeholder subnets (no IP pool) and a .bridged/.bridgedViaHelper NetworkVariant for the plugin helper. The BridgeInterfaceStrategy communicates XPC additional data payload forwarded by the BridgedVmnetNetwork to the concrete network interface. BridgeInterfaceStrategy maps: - .bridged network attachments to a BridgedNetworkInterface for the given host interface. - .bridgedViaHelper network attachments to a FileHandleNetworkInterface for the FileHandle endpoint where the plugin helper will provide bridging services to the runtime. Co-authored-by: Curd Becker --- .../NetworkVmnetHelper+Start.swift | 2 + .../RuntimeLinuxHelper+Start.swift | 2 + .../Network/Client/BridgeNetworkKeys.swift | 23 +++ .../Network/Client/NetworkVariant.swift | 2 + .../Server/BridgedVmnetNetwork.swift | 155 ++++++++++++++++++ .../Server/BridgedInterfaceStrategy.swift | 56 +++++++ .../RuntimeLinux/Server/RuntimeService.swift | 4 +- 7 files changed, 242 insertions(+), 2 deletions(-) create mode 100644 Sources/Services/Network/Client/BridgeNetworkKeys.swift create mode 100644 Sources/Services/NetworkVmnet/Server/BridgedVmnetNetwork.swift create mode 100644 Sources/Services/RuntimeLinux/Server/BridgedInterfaceStrategy.swift diff --git a/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift b/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift index 08005f6ba..bbd1b35ab 100644 --- a/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift +++ b/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift @@ -141,6 +141,8 @@ extension NetworkVmnetHelper { ) } return try ReservedVmnetNetwork(configuration: configuration, log: log) + case .bridged, .bridgedViaHelper: + return try BridgedVmnetNetwork(configuration: configuration, log: log) } } } diff --git a/Sources/Plugins/RuntimeLinux/RuntimeLinuxHelper+Start.swift b/Sources/Plugins/RuntimeLinux/RuntimeLinuxHelper+Start.swift index 3c7938b8e..796bf2aff 100644 --- a/Sources/Plugins/RuntimeLinux/RuntimeLinuxHelper+Start.swift +++ b/Sources/Plugins/RuntimeLinux/RuntimeLinuxHelper+Start.swift @@ -68,6 +68,8 @@ extension RuntimeLinuxHelper { NetworkInterfaceKey(plugin: "container-network-vmnet", variant: "allocationOnly"): IsolatedInterfaceStrategy() ] if #available(macOS 26, *) { + interfaceStrategies[NetworkInterfaceKey(plugin: "container-network-vmnet", variant: "bridged")] = BridgedInterfaceStrategy(log: log) + interfaceStrategies[NetworkInterfaceKey(plugin: "container-network-vmnet", variant: "bridgedViaHelper")] = BridgedInterfaceStrategy(log: log) interfaceStrategies[NetworkInterfaceKey(plugin: "container-network-vmnet", variant: "reserved")] = NonisolatedInterfaceStrategy(log: log) } diff --git a/Sources/Services/Network/Client/BridgeNetworkKeys.swift b/Sources/Services/Network/Client/BridgeNetworkKeys.swift new file mode 100644 index 000000000..fd600f9bd --- /dev/null +++ b/Sources/Services/Network/Client/BridgeNetworkKeys.swift @@ -0,0 +1,23 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +public enum BridgeNetworkKeys: String { + case hostInterface + case enableTso + case enableChecksumOffload + case bufferedPacketCount + case sandboxEndpoint +} diff --git a/Sources/Services/Network/Client/NetworkVariant.swift b/Sources/Services/Network/Client/NetworkVariant.swift index 99a30b7b3..a557081a1 100644 --- a/Sources/Services/Network/Client/NetworkVariant.swift +++ b/Sources/Services/Network/Client/NetworkVariant.swift @@ -17,4 +17,6 @@ public enum NetworkVariant: String, Sendable { case reserved case allocationOnly + case bridged + case bridgedViaHelper } diff --git a/Sources/Services/NetworkVmnet/Server/BridgedVmnetNetwork.swift b/Sources/Services/NetworkVmnet/Server/BridgedVmnetNetwork.swift new file mode 100644 index 000000000..8ac016eea --- /dev/null +++ b/Sources/Services/NetworkVmnet/Server/BridgedVmnetNetwork.swift @@ -0,0 +1,155 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerNetworkClient +import ContainerNetworkServer +import ContainerResource +import ContainerXPC +import Containerization +import ContainerizationError +import ContainerizationExtras +import Foundation +import Logging +import Synchronization +import Virtualization +import XPC + +public final class BridgedVmnetNetwork: ContainerNetworkServer.Network { + // FIXME: NetworkPluginStatus requires non-optional ipv4Subnet/ipv4Gateway; use placeholder + // values until the type is refactored to make them optional. + private static let placeholderSubnet = try! CIDRv4("0.0.0.0/0") + private static let placeholderGateway = IPv4Address(0) + private static let placeholderSubnetv6 = try! CIDRv6("::/0") + + private let log: Logger + private let configuration: NetworkConfiguration + private let hostInterface: String + private let statusMutex: Mutex + private let enableTso: Bool? + private let enableChecksumOffload: Bool? + private let bufferedPacketCount: Int? + + public init(configuration: NetworkConfiguration, log: Logger) throws { + // TODO: ignore the network type for now - having both network type and plugin variant is a bit awkward... + // I imagine this will get resolved later on + // guard configuration.mode == .bridge else { + // throw ContainerizationError(.unsupported, message: "invalid network mode \(configuration.mode)") + // } + guard let variantStr = configuration.options["variant"], + let variant = NetworkVariant(rawValue: variantStr), + variant == .bridged || variant == .bridgedViaHelper + else { + throw ContainerizationError( + .unsupported, + message: "invalid network variant \(configuration.options["variant"] ?? "unspecified")") + } + + guard let hostInterface = configuration.options["hostInterface"] else { + throw ContainerizationError(.invalidArgument, message: "hostInterface must be given as a plugin option") + } + let available = VZBridgedNetworkInterface.networkInterfaces.map { $0.identifier } + guard available.contains(hostInterface) else { + let list = available.isEmpty ? "none available" : available.joined(separator: ", ") + throw ContainerizationError(.invalidArgument, message: "no host interface '\(hostInterface)'; available: \(list)") + } + self.hostInterface = hostInterface + + self.configuration = configuration + self.log = log + self.statusMutex = Mutex(nil) + + if variant == .bridged { + self.enableTso = nil + self.enableChecksumOffload = nil + self.bufferedPacketCount = nil + } else { + let rawEnableTso = configuration.options[BridgeNetworkKeys.enableTso.rawValue] + if rawEnableTso != nil { + guard let enableTso = Bool(rawEnableTso!) else { + throw ContainerizationError(.invalidArgument, message: "enableTso must be a boolean") + } + self.enableTso = enableTso + } else { + self.enableTso = nil + } + + let rawEnableChecksumOffload: String? = configuration.options[BridgeNetworkKeys.enableChecksumOffload.rawValue] + if rawEnableChecksumOffload != nil { + guard let enableChecksumOffload = Bool(rawEnableChecksumOffload!) else { + throw ContainerizationError(.invalidArgument, message: "enableChecksumOffload must be a boolean") + } + self.enableChecksumOffload = enableChecksumOffload + } else { + self.enableChecksumOffload = nil + } + + let rawBufferedPacketCount = configuration.options[BridgeNetworkKeys.bufferedPacketCount.rawValue] + if rawBufferedPacketCount != nil { + guard let bufferedPacketCount = Int(rawBufferedPacketCount!), bufferedPacketCount > 0 else { + throw ContainerizationError(.invalidArgument, message: "bufferedPacketCount must be a positive integer") + } + self.bufferedPacketCount = bufferedPacketCount + } else { + self.bufferedPacketCount = nil + } + } + } + + public nonisolated var id: String { configuration.id } + + public var status: NetworkStatus? { + self.statusMutex.withLock { $0 } + } + + public nonisolated func withAdditionalData(_ handler: (XPCMessage?) throws -> Void) throws { + let bridgeConfigMsg = XPCMessage(object: xpc_dictionary_create_empty()) + bridgeConfigMsg.set(key: BridgeNetworkKeys.hostInterface.rawValue, value: hostInterface) + if self.enableTso != nil { + bridgeConfigMsg.set(key: BridgeNetworkKeys.enableTso.rawValue, value: self.enableTso!) + } + if self.enableChecksumOffload != nil { + bridgeConfigMsg.set(key: BridgeNetworkKeys.enableChecksumOffload.rawValue, value: self.enableChecksumOffload!) + } + if self.bufferedPacketCount != nil { + bridgeConfigMsg.set(key: BridgeNetworkKeys.bufferedPacketCount.rawValue, value: UInt64(self.bufferedPacketCount!)) + } + + let msg: XPCMessage = XPCMessage(object: xpc_dictionary_create_empty()) + msg.set(key: NetworkKeys.additionalData.rawValue, value: bridgeConfigMsg.underlying) + try handler(msg) + } + + public func start() async throws { + try statusMutex.withLock { status in + + guard status == nil else { + throw ContainerizationError(.invalidArgument, message: "cannot start network \(configuration.id): already started") + } + + status = NetworkStatus( + ipv4Subnet: Self.placeholderSubnet, ipv4Gateway: Self.placeholderGateway, ipv6Subnet: Self.placeholderSubnetv6 + ) + + log.info( + "started bridged network", + metadata: [ + "id": "\(configuration.id)", + "hostInterface": "\(hostInterface)", + ] + ) + } + } +} diff --git a/Sources/Services/RuntimeLinux/Server/BridgedInterfaceStrategy.swift b/Sources/Services/RuntimeLinux/Server/BridgedInterfaceStrategy.swift new file mode 100644 index 000000000..29ff55f1b --- /dev/null +++ b/Sources/Services/RuntimeLinux/Server/BridgedInterfaceStrategy.swift @@ -0,0 +1,56 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerNetworkClient +import ContainerResource +import ContainerRuntimeClient +import ContainerXPC +import Containerization +import ContainerizationError +import Logging + +/// Interface strategy for containers that use macOS's custom network feature. +@available(macOS 26, *) +public struct BridgedInterfaceStrategy: InterfaceStrategy { + private let log: Logger + + public init(log: Logger) { + self.log = log + } + + public func toInterface(attachment: Attachment, interfaceIndex: Int, additionalData: XPCMessage?) throws -> Interface { + guard let bridgeDictionary = additionalData?.xpcDictionary(key: NetworkKeys.additionalData.rawValue) else { + throw ContainerizationError(.internalError, message: "did not receive bridge dictionary in interface additional message") + } + let bridgeData = XPCMessage(object: bridgeDictionary) + + guard let ifaceName = bridgeData.string(key: BridgeNetworkKeys.hostInterface.rawValue) + else { + throw ContainerizationError(.invalidState, message: "bridge network missing host interface name") + } + guard let containerBridgeEndpoint = bridgeData.fileHandle(key: BridgeNetworkKeys.sandboxEndpoint.rawValue) else { + return BridgedNetworkInterface( + hostInterfaceName: ifaceName, + macAddress: attachment.macAddress + ) + } + + return FileHandleNetworkInterface( + fileHandle: containerBridgeEndpoint, + macAddress: attachment.macAddress + ) + } +} diff --git a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift index 3fc084821..356adb0a7 100644 --- a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift +++ b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift @@ -267,10 +267,10 @@ public actor RuntimeService { // NOTE: We can support a user providing new entries eventually, but for now craft // a default /etc/hosts. var hostsEntries = [Hosts.Entry.localHostIPV4()] - if !interfaces.isEmpty, let primaryIfaceAddr = interfaces[0].ipv4Address { + if !interfaces.isEmpty, let ipv4Address = interfaces[0].ipv4Address, !ipv4Address.address.isUnspecified { hostsEntries.append( Hosts.Entry( - ipAddress: primaryIfaceAddr.address.description, + ipAddress: ipv4Address.address.description, hostnames: [czConfig.hostname ?? id], )) } From 6b7470d71c0f85f23f8cbbf55eb3ce9f3ed47d4f Mon Sep 17 00:00:00 2001 From: Curd Becker Date: Mon, 1 Jun 2026 01:52:43 +0200 Subject: [PATCH 7/7] Add a BridgeNetworkService for socket-based bridge forwarding The BridgeNetworkService is heavily inspired by nirs/vmnet-helper and draws on the same concept. This allow bridging to a real host network interface without having a binary signed with the com.apple.vm.networking entitlement, e.g. during container development. Therefore, the service allocates a standalone vmnet interface and a dedicate UNIX datagram socket pair. One end of the pair remains local to the service as the network attachment of the runtime, the other end is sent to the runtime where it is assigned in the underlying VM configuration as `VZFileHandleNetworkDeviceAttachment`. This setup allows the service to perform manual bridging between the vmnnet network interface attached to the host interface and the runtime using event handlers on both sides. While this is obviously certainly less performant and generally inferior to having the same capabilities integrated directly in the Virtualization Framework, it provides a very similar setting and is actually possible without root privileges or any special entitlements. --- .../NetworkVmnetHelper+Start.swift | 11 +- .../Network/Server/BridgeNetworkService.swift | 488 ++++++++++++++++++ 2 files changed, 498 insertions(+), 1 deletion(-) create mode 100644 Sources/Services/Network/Server/BridgeNetworkService.swift diff --git a/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift b/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift index bbd1b35ab..b759b8a60 100644 --- a/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift +++ b/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift @@ -104,7 +104,7 @@ extension NetworkVmnetHelper { log: log ) try await network.start() - let service = try await DefaultNetworkService(network: network, log: log) + let service: NetworkService = try await Self.createNetworkService(network: network, variant: variant, log: log) let harness = NetworkHarness(service: service) let xpc = XPCServer( identifier: serviceIdentifier, @@ -129,6 +129,15 @@ extension NetworkVmnetHelper { } } + private static func createNetworkService(network: Network, variant: NetworkVariant, log: Logger) async throws -> NetworkService { + switch variant { + case .bridged, .bridgedViaHelper: + return try await BridgeNetworkService(network: network, variant: variant, log: log) + default: + return try await DefaultNetworkService(network: network, log: log) + } + } + private static func createNetwork(configuration: NetworkConfiguration, variant: NetworkVariant, log: Logger) throws -> Network { switch variant { case .allocationOnly: diff --git a/Sources/Services/Network/Server/BridgeNetworkService.swift b/Sources/Services/Network/Server/BridgeNetworkService.swift new file mode 100644 index 000000000..ebf6b629c --- /dev/null +++ b/Sources/Services/Network/Server/BridgeNetworkService.swift @@ -0,0 +1,488 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerNetworkClient +import ContainerResource +import ContainerXPC +import ContainerizationError +import ContainerizationExtras +import Foundation +import Logging +import vmnet + +internal final class Bridge { + + internal final class PacketBuffer { + public let buffers: UnsafeMutableRawPointer + public let iovs: UnsafeMutablePointer + public let packets: UnsafeMutablePointer + + public let maxPacketSize: Int + public let maxPktCount: Int + + public init(maxPktCount: Int, maxPktSize: Int) { + self.buffers = .allocate(byteCount: maxPktCount * maxPktSize, alignment: 16) + self.iovs = .allocate(capacity: maxPktCount) + self.packets = .allocate(capacity: maxPktCount) + for i in 0..sandbox + private let hostPacketBuffer: PacketBuffer + // sandbox->host + private let sandboxPacketBuffer: PacketBuffer + + public init( + identifier: String, + hostInterface: String, + logger: Logger, + enableTso: Bool, + enableChecksumOffload: Bool, + bufferedPacketCount: Int + ) throws { + var fds: [Int32] = [-1, -1] + let rc = fds.withUnsafeMutableBufferPointer { + socketpair(AF_UNIX, SOCK_DGRAM, 0, $0.baseAddress) + } + guard rc == 0 else { + throw ContainerizationError(.internalError, message: "unable to create socket pairs for UNIX bridge endpoints") + } + + let networkEndpoint = FileHandle(fileDescriptor: fds[0], closeOnDealloc: true) + let sandboxEndpoint = FileHandle(fileDescriptor: fds[1], closeOnDealloc: true) + + // Make our endpoint side non blocking, so the read source can drain it in a loop. + let flags = fcntl(networkEndpoint.fileDescriptor, F_GETFL) + _ = fcntl(networkEndpoint.fileDescriptor, F_SETFL, flags | O_NONBLOCK) + + // vmnet interface descriptor. + let desc = xpc_dictionary_create(nil, nil, 0) + xpc_dictionary_set_uint64( + desc, vmnet_operation_mode_key, + UInt64(vmnet.operating_modes_t.VMNET_BRIDGED_MODE.rawValue) + ) + xpc_dictionary_set_string(desc, vmnet_shared_interface_name_key, hostInterface) + xpc_dictionary_set_bool(desc, vmnet_enable_tso_key, enableTso) + xpc_dictionary_set_bool( + desc, vmnet_enable_checksum_offload_key, + enableChecksumOffload) + + // Unfortunately, apparently not understood currently - otherwise this would maybe provide us the opportunity to + // set a custom MAC address + // xpc_dictionary_set_bool(desc, vmnet_allocate_mac_address_key, false) + + // start interface synchronously — vmnet posts the completion on `queue`. + let queue = DispatchQueue(label: identifier, qos: .userInitiated) + let sema = DispatchSemaphore(value: 0) + var startStatus: vmnet_return_t = .VMNET_FAILURE + var maxPacketSize = 0 + var mtu: UInt32 = 0 + var mac = "" + + let iface = vmnet_start_interface(desc, queue) { status, param in + startStatus = status + if status == .VMNET_SUCCESS, let param = param { + maxPacketSize = Int(xpc_dictionary_get_uint64(param, vmnet_max_packet_size_key)) + mtu = UInt32(xpc_dictionary_get_uint64(param, vmnet_mtu_key)) + if let cstr = xpc_dictionary_get_string(param, vmnet_mac_address_key) { + mac = String(cString: cstr) + } + } + sema.signal() + } + sema.wait() + + guard startStatus == .VMNET_SUCCESS, let iface = iface else { + throw ContainerizationError(.internalError, message: "unable to start vmnet bridge interface") + } + guard let macAddress = try? MACAddress(mac) else { + throw ContainerizationError(.internalError, message: "vmnet returned an invalid mac address for vmnet bridge interface") + } + + self.log = logger + + self.identifier = identifier + self.networkEndpoint = networkEndpoint + self.sandboxEndpoint = sandboxEndpoint + self.macAddress = macAddress + self.mtu = mtu + self.interfaceRef = iface + self.queue = queue + + self.hostPacketBuffer = PacketBuffer(maxPktCount: bufferedPacketCount, maxPktSize: maxPacketSize) + self.sandboxPacketBuffer = PacketBuffer(maxPktCount: 1, maxPktSize: maxPacketSize) + } + + public func start() throws { + guard self.readSource == nil, !stopped else { + throw ContainerizationError(.invalidState, message: "bridge was already started") + } + + // TODO: despite Swift's claim that there are no function calls inside the event handlers that could throw exceptions, + // this is unfortunately simply not true. this is possibly connected due to some Objective-C legacy behavior + // of the vmnet_interface API. + // further investigation in catching these exceptions and making them visible might prove useful, since + // otherwise the network service can silently crash without any trace in the logs. the sandbox service will + // also keep running and will only notice that its interface has lost its carrier (quite fitting). only the + // console logging does provide then a crash dump including stack trace featuring some message like this: + // + // *** Terminating app due to uncaught exception 'NSFileHandleOperationException', reason: '*** -[NSConcreteFileHandle fileDescriptor]: Bad file descriptor' + + // host -> vm: vmnet fires an event when packets are buffered. + let st = vmnet_interface_set_event_callback( + interfaceRef, .VMNET_INTERFACE_PACKETS_AVAILABLE, queue + ) { [weak self] _, _ in + self?.relayFromHostToSandbox() + } + guard st == .VMNET_SUCCESS else { + throw ContainerizationError(.internalError, message: "vmnet_interface_set_event_callback failed") + } + + // vm -> host: dispatch source on the socket. + let src = DispatchSource.makeReadSource(fileDescriptor: networkEndpoint.fileDescriptor, queue: queue) + src.setEventHandler { [weak self] in self?.relayFromSandboxToHost() } + src.resume() + self.readSource = src + } + + public func stop() throws { + guard !stopped else { + throw ContainerizationError(.invalidState, message: "bridge was already stopped") + } + stopped = true + + guard let readSource else { + throw ContainerizationError(.internalError, message: "read source for network endpoint has not been created") + } + readSource.cancel() + + let sema = DispatchSemaphore(value: 0) + let st = vmnet_stop_interface(interfaceRef, queue, { _ in sema.signal() }) + sema.wait() + + guard st == .VMNET_SUCCESS else { + throw ContainerizationError(.internalError, message: "failed to stop vmnet bridge interface") + } + } + + // bridge input and output helpers + private func readFromHost() -> Int32 { + hostPacketBuffer.reset() + var count = Int32(hostPacketBuffer.maxPktCount) + let st = vmnet_read(interfaceRef, hostPacketBuffer.packets, &count) + guard st == .VMNET_SUCCESS, count > 0 else { + return -1 + } + return count + } + + private func writeToSandbox(count: Int) -> Int { + var written = 0 + for i in 0.. 0 else { + log.warning("received invalid number of packets from host: \(pktsIn)") + return + } + let pktsOut = writeToSandbox(count: pktsIn) + guard pktsOut == pktsIn else { + log.warning("could not forward all packets to sandbox: \(pktsIn) != \(pktsOut)") + return + } + } + + private func relayFromSandboxToHost() { + // vmnet-helper uses Apple-private system calls sendmsg_x/recmsg_x to forward multiple packets + // in a single system call on the bridge... that sounds intuitively as it will likely achieve quite a + // performance improvement, but it also might break without further notice + // + // in doubt: this is just a fallback for the likely even more optimized internal VirtualizationFramework + // bridging functionality + let pktSize = read( + networkEndpoint.fileDescriptor, sandboxPacketBuffer.buffers, + sandboxPacketBuffer.maxPacketSize) + guard pktSize > 0 else { + log.warning("failed to read from sandbox socket: \(pktSize) - EOF?") + return + } + + sandboxPacketBuffer.iovs.pointee.iov_len = pktSize + sandboxPacketBuffer.packets.pointee.vm_pkt_size = pktSize + sandboxPacketBuffer.packets.pointee.vm_flags = 0 + + var pktCount: Int32 = 1 + guard vmnet_write(interfaceRef, sandboxPacketBuffer.packets, &pktCount) == .VMNET_SUCCESS, pktCount == 1 else { + log.warning("failed to forward packet from sandbox to host") + return + } + } + +} + +public actor BridgeNetworkService: NetworkService { + + private let network: any Network + private let variant: NetworkVariant + private let log: Logger + private var allocationsBySession: [XPCServerSession: [(hostname: String, bridge: Bridge?)]] + private var macAddresses: [String: MACAddress] + + /// Set up a network service for the specified network. + public init( + network: any Network, + variant: NetworkVariant, + log: Logger + ) async throws { + guard await network.status != nil else { + throw ContainerizationError(.invalidState, message: "network \(network.id) must be running") + } + + self.network = network + self.variant = variant + self.log = log + self.allocationsBySession = [:] + self.macAddresses = [:] + } + + @Sendable + public func status() async throws -> NetworkStatus { + guard let status = await network.status else { + throw ContainerizationError(.invalidState, message: "network \(network.id) is not running") + } + return status + } + + @Sendable + public func allocate( + hostname: String, + macAddress: MACAddress?, + session: XPCServerSession + ) async throws -> (attachment: Attachment, additionalData: XPCMessage?) { + log.debug("enter", metadata: ["func": "\(#function)"]) + defer { log.debug("exit", metadata: ["func": "\(#function)"]) } + + guard await network.status != nil else { + throw ContainerizationError(.invalidState, message: "network \(network.id) must be running") + } + + // retrieve configuration from the network + // this is a bit awkward compared to the regular purpose of the network in managing allocations, since this is all + // handled externally by the network infrastructure on the other side of the bridge. therefore the network is for us + // simply a quite overblown struct/adapter that passes information along that are still unknown during creation time. + var additionalData: XPCMessage? + try network.withAdditionalData { + additionalData = $0 + } + + guard let bridgeDictionary = additionalData?.xpcDictionary(key: NetworkKeys.additionalData.rawValue) else { + throw ContainerizationError(.internalError, message: "did not receive bridge dictionary in network additional message") + } + let bridgeMessage = XPCMessage(object: bridgeDictionary) + + guard let hostInterface = bridgeMessage.string(key: BridgeNetworkKeys.hostInterface.rawValue) + else { + throw ContainerizationError(.invalidState, message: "bridge network is not assigned to a host interface") + } + + var attachment: Attachment + var bridge: Bridge? + if variant == .bridged { + let macAddress = macAddress ?? MACAddress((UInt64.random(in: 0...UInt64.max) & 0x0cff_ffff_ffff) | 0xf200_0000_0000) + attachment = Attachment( + network: network.id, + hostname: hostname, + ipv4Address: nil, + ipv4Gateway: nil, + ipv6Address: nil, + macAddress: macAddress, + mtu: nil, + ) + bridge = nil + } else { + let enableTso = + bridgeMessage.dataNoCopy(key: BridgeNetworkKeys.enableTso.rawValue) != nil ? bridgeMessage.bool(key: BridgeNetworkKeys.enableTso.rawValue) : nil + let enableChecksumOffload = + bridgeMessage.dataNoCopy(key: BridgeNetworkKeys.enableChecksumOffload.rawValue) != nil + ? bridgeMessage.bool(key: BridgeNetworkKeys.enableChecksumOffload.rawValue) : nil + let batchSize = + bridgeMessage.dataNoCopy(key: BridgeNetworkKeys.bufferedPacketCount.rawValue) != nil + ? Int(bridgeMessage.uint64(key: BridgeNetworkKeys.bufferedPacketCount.rawValue)) : nil + + bridge = try! Bridge( + identifier: "vmnet-bridge-\(network.id)-\(hostname)", hostInterface: hostInterface, logger: log, enableTso: enableTso ?? false, + enableChecksumOffload: enableChecksumOffload ?? false, bufferedPacketCount: batchSize ?? 64) + try bridge!.start() + + attachment = Attachment( + network: network.id, + hostname: hostname, + ipv4Address: nil, + ipv4Gateway: nil, + ipv6Address: nil, + macAddress: bridge!.macAddress, + mtu: bridge!.mtu + ) + + // add the sandbox-side fd endpoint to the bridge message + bridgeMessage.set(key: BridgeNetworkKeys.sandboxEndpoint.rawValue, value: bridge!.sandboxEndpoint) + + // close the sandbox endpoint's fd, since we do not need it anymore and it has been just dup-ed + // when we attached it to the XPC message above + try bridge!.sandboxEndpoint.close() + } + + log.info( + "allocated attachment", + metadata: [ + "hostname": "\(hostname)", + "hostInterface": "\(hostInterface)", + "macAddress": "\(attachment.macAddress!)", + "bridge": "\(bridge?.identifier ?? "vf-based")", + ]) + + if allocationsBySession[session] == nil { + allocationsBySession[session] = [] + await session.onDisconnect { [weak self] in + await self?.releaseSession(session) + } + } + allocationsBySession[session]!.append((hostname: hostname, bridge: bridge)) + macAddresses[hostname] = attachment.macAddress + + return (attachment: attachment, additionalData: additionalData) + } + + private func releaseSession(_ session: XPCServerSession) async { + guard let allocations = allocationsBySession.removeValue(forKey: session) else { + return + } + for allocation: (hostname: String, bridge: Bridge?) in allocations { + macAddresses[allocation.hostname] = nil + guard let bridge = allocation.bridge else { + continue + } + do { + try bridge.stop() + } catch let error as ContainerizationError { + log.error( + "failed to stop bridge for attachment", + metadata: [ + "error": Logger.MetadataValue.string(error.message) + ]) + } catch { + assert(false, "should never happen") + } + } + log.info("released session", metadata: ["allocations": "\(allocations.count)"]) + } + + @Sendable + public func lookup(hostname: String) async throws -> Attachment? { + log.debug("enter", metadata: ["func": "\(#function)"]) + defer { log.debug("exit", metadata: ["func": "\(#function)"]) } + + guard await network.status != nil else { + throw ContainerizationError(.invalidState, message: "network \(network.id) must be running") + } + guard let macAddress = macAddresses[hostname] else { + log.warning("MAC address for hostname \(hostname) is not a valid attachment") + return nil + } + + let attachment = Attachment( + network: network.id, + hostname: hostname, + ipv4Address: nil, + ipv4Gateway: nil, + ipv6Address: nil, + macAddress: macAddress + ) + log.info( + "lookup attachment", + metadata: [ + "hostname": "\(hostname)", + "macAddress": "\(macAddress)", + ]) + + return attachment + } +}