Skip to content

[WIP, NOREVIEW] Linux SslStream: custom BIO_METHOD over managed buffer windows #128245

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
rzikm wants to merge 5 commits into
base: main
Choose a base branch
from
Draft

[WIP, NOREVIEW] Linux SslStream: custom BIO_METHOD over managed buffer windows #128245

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a134137
Linux SslStream: custom BIO over managed buffer windows
rzikm May 14, 2026
68743ca
Address PR review feedback
rzikm May 15, 2026
7371f2e
SslStream/Linux: decrypt directly into user buffer
rzikm May 15, 2026
dc16ab6
Revert SslStream/Linux: decrypt directly into user buffer
rzikm May 15, 2026
7d3fc43
SslStream/Linux: atomic SSL ops and direct-decrypt into user buffer
rzikm May 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
185 changes: 139 additions & 46 deletions src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.OpenSsl.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
using System.Security.Cryptography;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Threading;
using Microsoft.Win32.SafeHandles;

internal static partial class Interop
Expand Down Expand Up @@ -666,21 +667,51 @@ internal static SecurityStatusPal SslRenegotiate(SafeSslHandle sslContext, out b
return new SecurityStatusPal(SecurityStatusPalErrorCode.OK);
}

internal static SecurityStatusPalErrorCode DoSslHandshake(SafeSslHandle context, ReadOnlySpan<byte> input, ref ProtocolToken token)
internal static unsafe SecurityStatusPalErrorCode DoSslHandshake(SafeSslHandle context, ReadOnlySpan<byte> input, ref ProtocolToken token)
{
token.Size = 0;
Exception? handshakeException = null;

if (input.Length > 0)
// Reserve a reasonable initial window in the outgoing token; the spill buffer
// catches anything that doesn't fit.
const int InitialHandshakeWindow = 4096;
token.EnsureAvailableSpace(InitialHandshakeWindow);

// Drain any bytes accumulated in the OutputBio's spill from a prior call
// (e.g. SSL_read emitting alerts before this handshake step).
DrainOutputBioSpill(context, ref token);
token.EnsureAvailableSpace(InitialHandshakeWindow);

int retVal;
int writtenToWindow;
int spillLen;
Ssl.SslErrorCode errorCode;

Span<byte> windowSpan = token.AvailableSpan;
fixed (byte* inputPtr = input)
fixed (byte* windowPtr = windowSpan)
{
if (Ssl.BioWrite(context.InputBio!, ref MemoryMarshal.GetReference(input), input.Length) != input.Length)
if (input.Length > 0)
{
Ssl.BioSetReadWindow(context.InputBio!, inputPtr, input.Length);
}

Ssl.BioSetWriteWindow(context.OutputBio!, windowPtr, windowSpan.Length);

try
{
// Make sure we clear out the error that is stored in the queue
throw Crypto.CreateOpenSslCryptographicException();
retVal = Ssl.SslDoHandshake(context, out errorCode);
Ssl.BioGetWriteResult(context.OutputBio!, out writtenToWindow, out spillLen);
}
finally
{
Ssl.BioSetWriteWindow(context.OutputBio!, null, 0);
Ssl.BioClearReadWindow(context.InputBio!);
Comment thread
rzikm marked this conversation as resolved.
Outdated
}
}

int retVal = Ssl.SslDoHandshake(context, out Ssl.SslErrorCode errorCode);
token.Size += writtenToWindow;

if (retVal != 1)
{
if (errorCode == Ssl.SslErrorCode.SSL_ERROR_WANT_X509_LOOKUP)
Expand All @@ -706,31 +737,17 @@ internal static SecurityStatusPalErrorCode DoSslHandshake(SafeSslHandle context,
}
}

int sendCount = Crypto.BioCtrlPending(context.OutputBio!);
if (sendCount > 0)
if (spillLen > 0)
{
token.EnsureAvailableSpace(sendCount);
try
{
sendCount = BioRead(context.OutputBio!, token.AvailableSpan, sendCount);
}
catch (Exception) when (handshakeException != null)
{
// If we already have handshake exception, ignore any exception from BioRead().
}
finally
token.EnsureAvailableSpace(spillLen);
Span<byte> spillDst = token.AvailableSpan;
fixed (byte* spillPtr = spillDst)
{
if (sendCount <= 0)
{
// Make sure we clear out the error that is stored in the queue
Crypto.ErrClearError();
sendCount = 0;
}
int drained = Ssl.BioDrainSpill(context.OutputBio!, spillPtr, spillDst.Length);
token.Size += drained;
}
}

token.Size = sendCount;

if (handshakeException != null)
{
ExceptionDispatchInfo.Throw(handshakeException);
Expand All @@ -755,13 +772,45 @@ internal static SecurityStatusPalErrorCode DoSslHandshake(SafeSslHandle context,
return stateOk ? SecurityStatusPalErrorCode.OK : SecurityStatusPalErrorCode.ContinueNeeded;
}

internal static Ssl.SslErrorCode Encrypt(SafeSslHandle context, ReadOnlySpan<byte> input, ref ProtocolToken outToken)
internal static unsafe Ssl.SslErrorCode Encrypt(SafeSslHandle context, ReadOnlySpan<byte> input, ref ProtocolToken outToken)
{
int retVal = Ssl.SslWrite(context, ref MemoryMarshal.GetReference(input), input.Length, out Ssl.SslErrorCode errorCode);
// Drain any bytes that the OutputBio may have accumulated outside of an explicit
// write window (e.g. from a prior SSL_read that emitted alerts / KeyUpdate / etc.).
DrainOutputBioSpill(context, ref outToken);
Comment thread
rzikm marked this conversation as resolved.
Comment thread
rzikm marked this conversation as resolved.

// Preserve any bytes already in outToken (including those just drained from a prior SSL_read's
// alerts / KeyUpdate output). On error we restore Size to this snapshot so those bytes are
// still sent rather than overwritten with the partial output of a failed SSL_write.
int preWriteSize = outToken.Size;

// Worst-case TLS output for the user's plaintext.
int upperBound = ComputeMaxTlsOutput(input.Length);
outToken.EnsureAvailableSpace(upperBound);

int retVal;
int writtenToWindow;
int spillLen;
Ssl.SslErrorCode errorCode;

Span<byte> windowSpan = outToken.AvailableSpan;
fixed (byte* windowPtr = windowSpan)
{
Ssl.BioSetWriteWindow(context.OutputBio!, windowPtr, windowSpan.Length);
try
{
retVal = Ssl.SslWrite(context, ref MemoryMarshal.GetReference(input), input.Length, out errorCode);
Ssl.BioGetWriteResult(context.OutputBio!, out writtenToWindow, out spillLen);
}
finally
{
Ssl.BioSetWriteWindow(context.OutputBio!, null, 0);
}
}

if (retVal != input.Length)
{
outToken.Size = 0;
// Drop any partial output written by the failed SSL_write but keep the drained spill bytes.
outToken.Size = preWriteSize;
Comment on lines 801 to +804
switch (errorCode)
{
// indicate end-of-file
Expand All @@ -772,33 +821,77 @@ internal static Ssl.SslErrorCode Encrypt(SafeSslHandle context, ReadOnlySpan<byt
default:
throw new SslException(SR.Format(SR.net_ssl_encrypt_failed, errorCode), GetSslError(retVal, errorCode));
}

return errorCode;
}
else
{
int capacityNeeded = Crypto.BioCtrlPending(context.OutputBio!);
outToken.EnsureAvailableSpace(capacityNeeded);
retVal = BioRead(context.OutputBio!, outToken.AvailableSpan, capacityNeeded);

if (retVal <= 0)
{
// Make sure we clear out the error that is stored in the queue
Crypto.ErrClearError();
outToken.Size = 0;
}
else
outToken.Size += writtenToWindow;

if (spillLen > 0)
{
outToken.EnsureAvailableSpace(spillLen);
Span<byte> spillDst = outToken.AvailableSpan;
fixed (byte* spillPtr = spillDst)
{
outToken.Size = retVal;
int drained = Ssl.BioDrainSpill(context.OutputBio!, spillPtr, spillDst.Length);
outToken.Size += drained;
}
}

return errorCode;
}

internal static int Decrypt(SafeSslHandle context, Span<byte> buffer, out Ssl.SslErrorCode errorCode)
private static int ComputeMaxTlsOutput(int inputLength)
{
// TLS 1.3 record max plaintext = 16384 bytes. Per-record overhead is bounded by
// OpenSSL's SSL3_RT_MAX_ENCRYPTED_OVERHEAD (256 bytes, covering record header, AEAD
// tag, optional MAC, padding, and the inner content-type byte for TLS 1.3).
// Always add slack for at least one record's overhead even when inputLength == 0,
// since SSL_write of an empty buffer can still emit handshake/alert bytes.
const int MaxRecordOverhead = 256;
int records = (inputLength >> 14) + 2;
return inputLength + (records * MaxRecordOverhead);
}

private static unsafe void DrainOutputBioSpill(SafeSslHandle context, ref ProtocolToken outToken)
{
BioWrite(context.InputBio!, buffer);
Ssl.BioGetWriteResult(context.OutputBio!, out _, out int spillLen);
if (spillLen <= 0)
{
return;
}
outToken.EnsureAvailableSpace(spillLen);
Span<byte> dst = outToken.AvailableSpan;
fixed (byte* dstPtr = dst)
{
int drained = Ssl.BioDrainSpill(context.OutputBio!, dstPtr, dst.Length);
outToken.Size += drained;
}
}

internal static unsafe int Decrypt(SafeSslHandle context, ReadOnlySpan<byte> input, Span<byte> output, out Ssl.SslErrorCode errorCode)
{
Debug.Assert(output.Length > 0, "Decrypt output buffer must be non-empty");

int retVal;
fixed (byte* inputPtr = input)
{
if (input.Length > 0)
{
Ssl.BioSetReadWindow(context.InputBio!, inputPtr, input.Length);
}
try
{
retVal = Ssl.SslRead(context, ref MemoryMarshal.GetReference(output), output.Length, out errorCode);
}
finally
{
if (input.Length > 0)
{
Ssl.BioClearReadWindow(context.InputBio!);
}
}
}

int retVal = Ssl.SslRead(context, ref MemoryMarshal.GetReference(buffer), buffer.Length, out errorCode);
if (retVal > 0)
{
return retVal;
Expand Down
25 changes: 23 additions & 2 deletions src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.Ssl.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,9 @@ internal static unsafe ReadOnlySpan<byte> SslGetAlpnSelected(SafeSslHandle ssl)
[LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_SslWrite", SetLastError = true)]
internal static partial int SslWrite(SafeSslHandle ssl, ref byte buf, int num, out SslErrorCode error);

[LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_SslPending")]
internal static partial int SslPending(SafeSslHandle ssl);

[LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_SslRead", SetLastError = true)]
internal static partial int SslRead(SafeSslHandle ssl, ref byte buf, int num, out SslErrorCode error);

Expand Down Expand Up @@ -131,6 +134,24 @@ internal static ushort[] GetDefaultSignatureAlgorithms()
[LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_BioWrite")]
internal static partial int BioWrite(SafeBioHandle b, ref byte data, int len);

[LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_BioNewManagedSpan")]
internal static partial SafeBioHandle BioNewManagedSpan();

[LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_BioSetReadWindow")]
internal static unsafe partial void BioSetReadWindow(SafeBioHandle bio, byte* ptr, int len);

[LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_BioClearReadWindow")]
internal static partial void BioClearReadWindow(SafeBioHandle bio);

[LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_BioSetWriteWindow")]
internal static unsafe partial void BioSetWriteWindow(SafeBioHandle bio, byte* ptr, int capacity);

[LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_BioGetWriteResult")]
internal static partial void BioGetWriteResult(SafeBioHandle bio, out int writtenToWindow, out int spillLen);

[LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_BioDrainSpill")]
internal static unsafe partial int BioDrainSpill(SafeBioHandle bio, byte* dst, int dstLen);

[LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_SslGetPeerCertificate")]
internal static partial IntPtr SslGetPeerCertificate(SafeSslHandle ssl);

Expand Down Expand Up @@ -437,8 +458,8 @@ internal void MarkHandshakeCompleted()

public static SafeSslHandle Create(SafeSslContextHandle context, SslAuthenticationOptions options)
{
SafeBioHandle readBio = Interop.Crypto.CreateMemoryBio();
SafeBioHandle writeBio = Interop.Crypto.CreateMemoryBio();
SafeBioHandle readBio = Interop.Ssl.BioNewManagedSpan();
SafeBioHandle writeBio = Interop.Ssl.BioNewManagedSpan();
SafeSslHandle handle = Interop.Ssl.SslCreate(context);
if (readBio.IsInvalid || writeBio.IsInvalid || handle.IsInvalid)
{
Expand Down
Loading
Loading