From d0464621b06757b01cb1f265f9309654383c2f80 Mon Sep 17 00:00:00 2001 From: Trim Bresilla Date: Wed, 26 Feb 2025 01:42:53 +0100 Subject: [PATCH 1/2] feat: introduce image segmantation based on CustomStencil - Add a new binary asset `PP_Segmentation.uasset` - Introduce a new `SEGMENT` camera type with corresponding setup for capture source and post-process material - Enable temporal anti-aliasing for the scene capture component - Adjust the capture source and data formats for RGB and Depth camera types - Provide uninitialized data buffers for all camera types - Add a conditional check for `PostProcessMaterial` to ensure it's not null before use - Update the logic to consider the new `SEGMENT` camera type during data capture and processing - Add the option to disable rendering or publishing via new boolean properties, `Render` and `Publish` --- Content/Materials/PP_Segmentation.uasset | Bin 0 -> 7329 bytes .../Private/Sensors/RRROS2CameraComponent.cpp | 81 ++++++++++++------ .../Public/Sensors/RRROS2CameraComponent.h | 8 +- 3 files changed, 61 insertions(+), 28 deletions(-) create mode 100644 Content/Materials/PP_Segmentation.uasset diff --git a/Content/Materials/PP_Segmentation.uasset b/Content/Materials/PP_Segmentation.uasset new file mode 100644 index 0000000000000000000000000000000000000000..5db2eb5557b0b46ee9e49ed204485abcaf2a4174 GIT binary patch literal 7329 zcmbUm4Oml0a+B~csKFo<{}mx13L-y(qCiN5lt4mITko#;Odj$uc?ro20#{qKRy{m> zhhnv8Ed>Ph&Y$+MTCJ^GYwx_zD;o3^J=7Cz)nZ#u)Pg;3=jA;N6s_KTF1wkXnVp%P zowqw4eIsn!!=9d=g(8H!gb4kBdjJpI-dp?7_mR@J#{0sn^`VCorbl4h%%iWwb*Htv zeZLazd#x}m*ahQO1&ms_GE4vGHv90M=f1qW^V=&Q2Sy|T-aiVfDWqx>ggT*@n3&MQ=hgBt(ryRyVz ze;L;8$lu$3Go|w0pw_81$%hnEq!{;V07B8YkH{v>B{qi8QbwDBU?>xI#)|BHj*^M5@0Tb=HcdwZ3g(<3!!nikBo_ojgL!?jaNjc#79QS<5J`? zanq*8#6`s<$H&J+^%A?{(ZxXsxpT~ieAp>;4foKj!x1Vey8ZW&it81>-}Gja`P3TK z^~Nfrr^5Y{uj-;#2_?a=3NQ=EVHH8(`JLf40$G z6^OBmWCX1utV>9J3Yl!R+Ki+=%S;s7NECLgnivKYX@r$`cm`2ITC8Z+x5_m}G(0Y>*l@`6eek?beZKD~=NY0}yjEykl zkj0G6LT1?*vyDOi%eO2U!qI7UBu&ap29pKd%87p9&!OjGYslhs8V7{ggrfvcSB;FE z58IbwAZfi7&0l){b=brdlgZF){!o{_FHC{spI>&i{BrR$(v^Ot0&@8`^#X-4!O9Q4@a}m^l{0YRxGzV@KxI%0GrPHpfQkkxbo{4|?J- z=+$VuJyr=j#)a$|wJ8raJcC-C&rtMYmhbf)HFDs~plFh?ID$(evno6>j`ud|*n4;C zC?|xB(B+f*-hGwRjHLughIGvU`*|*IkJ18)hdc8Q=>6T-j_rce#PdETazw&Q=(GE+ zSDr@kGV-|n*=jfvKbOH8dfy-IhTZ334Vao0l$8=PsP zNpw&2g%2=hEh@kv+e?#t@h`W)y=p?Y1P5J)NkJRVHs{0d=3;W)Sb zE^{HI^a0Z(ccg~+*$T&C8Z9=Tu&6DRkz#N{fUQZV^<*(eW`DGE47k}r#v#|--1TQ%RCj+~Iw-GImic10IwWd6#-$wlE#ob*HpjtCwv6Ayl2K*i2EaTVNo4yGy zAMBaN`AEP0lZ~t}_;Bm`MWF*iIABrJhekve}Si2==JAmo;sGedLZDvd&rK z2cHh^w~D8GH@uEy@%=7n-6uXx7!CA&@%_Ge=iWZ@{)?CjK;`IA7ry0axN#)cL)V|~ zM@$EK3*eXNG96^UNSF1Dbn2(jWpi`T@j4HrpEF&?0P_Be?Q;@^a=0Jxf$ZYOa2tFQ z!0i+o02b9p7T|tz!U5e}Kw^L}u(KnCiXgV00K9CN9!UP9e)7S$a8re-8qP7NLo#s! zXPM69f#i4hlMj0WJRtuRCo^{hH{d#Fna<;Z*(=8n2Im_h8*{;7V zehy^8y8xbHFovF&(?=Hkz(UHfaXsgIuArexMY;kBFj@c%Jr|J-4f62t@^ByI<>}?) zJ;+Bq(pNlWhPCCkSX%#hMprx>;XE71mlrVED(tWJ;xEwKmu0*3)m1A1VWLE ztDC!rr`I4%ux}_53Pd8Ii^$d01$#lT49_DMv8(U6smX4`G=%&3LP^x}iuXMxqaAI(XG&C$aCN^$*e1a@hu9%aSuGHqtoj3obmlx>tWZvR@ zs=&(Fik23al)ka@%~h+{{H}cchRTg^Z`%A$RZVT(_WFj#9Xmh#OVhrO_8<88;IYq- ze{tf=zy9sy8GFmwbN@Vlq4ny&ueD#l@%1-1Z~buhKleKCKX~|K7uW@M@-U!XVr-Yt z#YN=e4t5EIOK}s4U0laab@NTuxD$oL#z!spkfc<+|G`nu3DdH__giE+?KM0o`s&0V zz%?*N^8ozbjz+~HD26N?NU=N72G`YuIiP_ zTPSoF9?+HOB&P6ttutM-r>&uVaDCbEa=Np#wzR8RB&{`f)|#tb8}=yw5HtR#R1FV3c3*>6?nvN#fIT3Cdafpdg{*wGK2{qDk_mn~t?r&Y)*9_00mgME$hpvNlBb zqkYSEnR0|^RHL9pBS-P2_FFOCXD1>F{t>sZHorlWkf@MRyJzgC*`UWTQ4&$4C*+s?#7lElYBL*OVUeH!Pya8 zhh<(03yv>9O5Rh?*iZaN`>EKfhLNQ$TU02vyvVMwC;8wFL?Oa=lr5D=gWdEzE$5`A zx+VJIS3zo*ChUl5)V;da0%eNXv)UtTfl|tb1BApgw~d(rfl8@x+2`lh9jhIMV~N`H zz$c-q2br5)qVWpg?&obYnVO2y>}EmoO}YDa6!kpnIMI66cCUC>l{r+~C=3fz)t*^b zd!}q`&Bj#$<;rK;-o9Z_X{+`uJ0g6$_7b%eN9{5kk>a%YQ&IKA#|44UEiZK5`fYBm ztYc;r))v;EP8xivAT}WWTdJkw&<2UX4~utwj^_2o0L2Dey~*_z*) zFW86k0Bu(eZu3F0@;kEw->S!v7%-Dza|?n{`lZ$&{VwHw`nTHqRmgwWtj%t1swr2K z2G?vkpgEuxHR8h(7JI8ex$a;U8{L8PMTtAY$eXJC?mgEX=lE#>s;yW$x>W6dZri*0 zf_petCg60?vaT5)BFrTSzIt16RdYJYmmb}8#xq{h3fUlzx>rjzoKEr?6S3eo$Et98 zL)Z2lY$3B~s^$9aB(X;8=W}keE^!?Wo$NZB#QBs2;K)`tT|w1uD}%QUeXFJYW@fd= z{VMg;bMHZFJ+yk)h}FC1$kEI{D%=;S8qU1g-O`2rK-E$^q}3j*t@7@DqtZSr!dX{Z z?H0Gd7=O^7j~XKIAfZ|G4fckjyT0~|G&T<`)HGp}W-sc}d@!fLCP8I_X6=aY0@rW; z7aV43wR?lo7E`KiMD_de;GkLrF=y#)u={_f{RZ|$)N=u@B7F)cLouS?H6MMAUr}c6 zataIs(ZF=yx6c&*m4a|N7r`SC7apAog77R-;BqjDpvcj;D9M!^ACthF=B}1N>n# zGm?mXz@=vAvW2Nklf{VVOt!`OG!#uhjy9jr<60KTH6V6nttvN_%p+_DCWB&FPdU?A z5iBNkMky%LpihhnWf#k_5VX}`!N(^@26ycZW~7@asrUK>rR38UVcD zaK}#sm6QPi?)U&t66lz9{Dsg4;DyIHexhKm|5*p+0RT8>0K68wPkLV#pari1-?JP} zfB02{I2;7hfB;Vw0A3KR-oHWY0`RX?o(H2|mxZuX2;K%P^iJ-@FJI}CCadv8C^yOL zI+XkU#Hji3@T#kUSuYdzmo8Q;vF}0?I0oKBy-R}uyg@*d$PxAraVY7qz;DhIjai7F z*-tdajAcBxF+t%ME?-a-29F*+X^&rL!>7+|jI&{&JQy1V0}SJE`^En|JBt6aant?n dWm9GxKMm*0>G2CPP*K}-jmyc;LivS&{(olr#moQz literal 0 HcmV?d00001 diff --git a/Source/RapyutaSimulationPlugins/Private/Sensors/RRROS2CameraComponent.cpp b/Source/RapyutaSimulationPlugins/Private/Sensors/RRROS2CameraComponent.cpp index 7358cb74..53722174 100644 --- a/Source/RapyutaSimulationPlugins/Private/Sensors/RRROS2CameraComponent.cpp +++ b/Source/RapyutaSimulationPlugins/Private/Sensors/RRROS2CameraComponent.cpp @@ -1,7 +1,6 @@ -// Copyright 2020-2023 Rapyuta Robotics Co., Ltd. + // Copyright 2020-2023 Rapyuta Robotics Co., Ltd. #include "Sensors/RRROS2CameraComponent.h" - #include "BufferVisualizationData.h" URRROS2CameraComponent::URRROS2CameraComponent() @@ -21,28 +20,56 @@ void URRROS2CameraComponent::PreInitializePublisher(UROS2NodeComponent* InROS2No { SceneCaptureComponent->FOVAngle = CameraComponent->FieldOfView; SceneCaptureComponent->OrthoWidth = CameraComponent->OrthoWidth; + SceneCaptureComponent->ShowFlags.SetTemporalAA(true); + + RenderTarget = NewObject(this, UTextureRenderTarget2D::StaticClass()); - // Set capture source and image properties based on camera type if (CameraType == EROS2CameraType::RGB) { - SceneCaptureComponent->CaptureSource = ESceneCaptureSource::SCS_FinalColorHDR; - RenderTarget = NewObject(this, UTextureRenderTarget2D::StaticClass()); + SceneCaptureComponent->CaptureSource = ESceneCaptureSource::SCS_SceneColorHDR; + + RenderTarget->RenderTargetFormat = ETextureRenderTargetFormat::RTF_RGBA8; RenderTarget->InitCustomFormat(Width, Height, EPixelFormat::PF_B8G8R8A8, true); Data.Encoding = TEXT("bgr8"); Data.Step = Width * 3; + Data.Data.AddUninitialized(Width * Height * 3); } else if (CameraType == EROS2CameraType::DEPTH) { CameraComponent->PostProcessSettings.WeightedBlendables.Array.Add( - FWeightedBlendable(1.0f, GetBufferVisualizationData().GetMaterial(TEXT("SceneDepth")))); + FWeightedBlendable(1.0f, GetBufferVisualizationData().GetMaterial(TEXT("SceneDepth"))) + ); SceneCaptureComponent->PostProcessSettings = CameraComponent->PostProcessSettings; SceneCaptureComponent->CaptureSource = ESceneCaptureSource::SCS_SceneDepth; - RenderTarget = NewObject(this, UTextureRenderTarget2D::StaticClass()); + RenderTarget->RenderTargetFormat = ETextureRenderTargetFormat::RTF_RGBA32f; RenderTarget->InitCustomFormat(Width, Height, EPixelFormat::PF_FloatRGBA, false); + Data.Encoding = TEXT("32FC1"); Data.Step = Width * 4; + Data.Data.AddUninitialized(Width * Height * 4); + } + else if (CameraType == EROS2CameraType::SEGMENT) + { + SceneCaptureComponent->CaptureSource = ESceneCaptureSource::SCS_FinalColorLDR; + SceneCaptureComponent->ShowFlags.SetPostProcessing(true); + + // material reference: /Script/Engine.Material'/RapyutaSimulationPlugins/Materials/PP_Segmentation.PP_Segmentation' + UMaterial* PostProcessMaterial = LoadObject(nullptr, TEXT("/RapyutaSimulationPlugins/Materials/PP_Segmentation.PP_Segmentation")); + + if(PostProcessMaterial){ // check nullptr + SceneCaptureComponent->AddOrUpdateBlendable(PostProcessMaterial); + } else { + UE_LOG(LogTemp, Log, TEXT("No PostProcessMaterial is assigend")); + } + + RenderTarget->RenderTargetFormat = ETextureRenderTargetFormat::RTF_RGBA8; + RenderTarget->InitCustomFormat(Width, Height, EPixelFormat::PF_B8G8R8A8, true); + + Data.Encoding = TEXT("bgr8"); + Data.Step = Width * 3; + Data.Data.AddUninitialized(Width * Height * 3); } // Common setup @@ -50,15 +77,13 @@ void URRROS2CameraComponent::PreInitializePublisher(UROS2NodeComponent* InROS2No Data.Header.FrameId = FrameId; Data.Width = Width; Data.Height = Height; - Data.Data.AddUninitialized(Width * Height * (CameraType == EROS2CameraType::RGB ? 3 : 4)); QueueSize = QueueSize < 1 ? 1 : QueueSize; // QueueSize should be more than 1 Super::PreInitializePublisher(InROS2Node, InTopicName); } void URRROS2CameraComponent::SensorUpdate() { - if (Render) - { + if (Render) { SceneCaptureComponent->CaptureScene(); CaptureNonBlocking(); } @@ -94,7 +119,7 @@ void URRROS2CameraComponent::CaptureNonBlocking() FIntRect rect(0, 0, renderTargetResource->GetSizeXY().X, renderTargetResource->GetSizeXY().Y); FReadSurfaceDataFlags flags(RCM_UNorm, CubeFace_MAX); - if (CameraType == EROS2CameraType::RGB) + if (CameraType == EROS2CameraType::RGB || CameraType == EROS2CameraType::SEGMENT) { FReadSurfaceContextRGB readSurfaceContext = {renderTargetResource, &(renderRequest->Image), rect, flags}; @@ -102,10 +127,11 @@ void URRROS2CameraComponent::CaptureNonBlocking() ( [readSurfaceContext, this](FRHICommandListImmediate& RHICmdList) { - RHICmdList.ReadSurfaceData(readSurfaceContext.SrcRenderTarget->GetRenderTargetTexture(), - readSurfaceContext.Rect, - *readSurfaceContext.OutData, - readSurfaceContext.Flags); + RHICmdList.ReadSurfaceData( + readSurfaceContext.SrcRenderTarget->GetRenderTargetTexture(), + readSurfaceContext.Rect, + *readSurfaceContext.OutData, + readSurfaceContext.Flags); }); } else if (CameraType == EROS2CameraType::DEPTH) @@ -116,10 +142,11 @@ void URRROS2CameraComponent::CaptureNonBlocking() ( [readSurfaceContext, this](FRHICommandListImmediate& RHICmdList) { - RHICmdList.ReadSurfaceFloatData(readSurfaceContext.SrcRenderTarget->GetRenderTargetTexture(), - readSurfaceContext.Rect, - *readSurfaceContext.Depth, - readSurfaceContext.Flags); + RHICmdList.ReadSurfaceFloatData( + readSurfaceContext.SrcRenderTarget->GetRenderTargetTexture(), + readSurfaceContext.Rect, + *readSurfaceContext.Depth, + readSurfaceContext.Flags); }); } @@ -140,29 +167,28 @@ void URRROS2CameraComponent::CaptureNonBlocking() FROSImg URRROS2CameraComponent::GetROS2Data() { - if (!RenderRequestQueue.IsEmpty()) - { + if (!RenderRequestQueue.IsEmpty() && (Publish == true)) { // Timestamp Data.Header.Stamp = URRConversionUtils::FloatToROSStamp(UGameplayStatics::GetTimeSeconds(GetWorld())); // Peek the next RenderRequest from queue FRenderRequest* nextRenderRequest = nullptr; RenderRequestQueue.Peek(nextRenderRequest); - if (nextRenderRequest && nextRenderRequest->RenderFence.IsFenceComplete()) + if (nextRenderRequest && nextRenderRequest->RenderFence.IsFenceComplete()) { - if (CameraType == EROS2CameraType::RGB) + if (CameraType == EROS2CameraType::RGB || CameraType == EROS2CameraType::SEGMENT) { // Process RGB data - for (int i = 0; i < nextRenderRequest->Image.Num(); i++) + for (int i = 0; i < nextRenderRequest->Image.Num(); i++) { Data.Data[i * 3 + 0] = nextRenderRequest->Image[i].B; Data.Data[i * 3 + 1] = nextRenderRequest->Image[i].G; Data.Data[i * 3 + 2] = nextRenderRequest->Image[i].R; } } - else if (CameraType == EROS2CameraType::DEPTH) + else if (CameraType == EROS2CameraType::DEPTH) { // Process Depth data - for (int i = 0; i < nextRenderRequest->Depth.Num(); i++) + for (int i = 0; i < nextRenderRequest->Depth.Num(); i++) { float value = nextRenderRequest->Depth[i].R.GetFloat() / 100; std::memcpy(&Data.Data[i * 4], &value, sizeof(value)); @@ -174,6 +200,7 @@ FROSImg URRROS2CameraComponent::GetROS2Data() QueueCount--; delete nextRenderRequest; } + } return Data; } @@ -181,4 +208,4 @@ FROSImg URRROS2CameraComponent::GetROS2Data() void URRROS2CameraComponent::SetROS2Msg(UROS2GenericMsg* InMessage) { CastChecked(InMessage)->SetMsg(GetROS2Data()); -} +} \ No newline at end of file diff --git a/Source/RapyutaSimulationPlugins/Public/Sensors/RRROS2CameraComponent.h b/Source/RapyutaSimulationPlugins/Public/Sensors/RRROS2CameraComponent.h index 1d49eed1..fc82aa84 100644 --- a/Source/RapyutaSimulationPlugins/Public/Sensors/RRROS2CameraComponent.h +++ b/Source/RapyutaSimulationPlugins/Public/Sensors/RRROS2CameraComponent.h @@ -38,7 +38,8 @@ UENUM(BlueprintType) enum class EROS2CameraType : uint8 { RGB UMETA(DisplayName = "RGB"), - DEPTH UMETA(DisplayName = "Depth") + DEPTH UMETA(DisplayName = "Depth"), + SEGMENT UMETA(DisplayName = "Segment") }; /** @@ -115,9 +116,14 @@ class RAPYUTASIMULATIONPLUGINS_API URRROS2CameraComponent : public URRROS2BaseSe UPROPERTY(EditAnywhere, BlueprintReadWrite) EROS2CameraType CameraType = EROS2CameraType::RGB; + // Allow to disable rendering (for performance, it still publishes still image) UPROPERTY(EditAnywhere, BlueprintReadWrite) bool Render = true; + // Allow to disable publishing (no image will be published) + UPROPERTY(EditAnywhere, BlueprintReadWrite) + bool Publish = true; + // ROS /** * @brief Update ROS 2 Msg structure from #RenderRequestQueue From 0d12831fa7f5d02414a9a1aae56d7b6a9fed3252 Mon Sep 17 00:00:00 2001 From: Trim Bresilla Date: Tue, 4 Mar 2025 20:35:37 +0100 Subject: [PATCH 2/2] feat: always enable image publishing in ROS2 Camera Component - Remove the condition check for Publishing in GetROS2Data to always proceed with the render request. - Remove the Publish boolean property from the RRROS2CameraComponent header, eliminating the ability to disable publishing images. --- .../Private/Sensors/RRROS2CameraComponent.cpp | 8 ++++---- .../Public/Sensors/RRROS2CameraComponent.h | 4 ---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/Source/RapyutaSimulationPlugins/Private/Sensors/RRROS2CameraComponent.cpp b/Source/RapyutaSimulationPlugins/Private/Sensors/RRROS2CameraComponent.cpp index 53722174..f6da4b49 100644 --- a/Source/RapyutaSimulationPlugins/Private/Sensors/RRROS2CameraComponent.cpp +++ b/Source/RapyutaSimulationPlugins/Private/Sensors/RRROS2CameraComponent.cpp @@ -1,4 +1,4 @@ - // Copyright 2020-2023 Rapyuta Robotics Co., Ltd. +// Copyright 2020-2023 Rapyuta Robotics Co., Ltd. #include "Sensors/RRROS2CameraComponent.h" #include "BufferVisualizationData.h" @@ -167,13 +167,13 @@ void URRROS2CameraComponent::CaptureNonBlocking() FROSImg URRROS2CameraComponent::GetROS2Data() { - if (!RenderRequestQueue.IsEmpty() && (Publish == true)) { + if (!RenderRequestQueue.IsEmpty()) { // Timestamp Data.Header.Stamp = URRConversionUtils::FloatToROSStamp(UGameplayStatics::GetTimeSeconds(GetWorld())); // Peek the next RenderRequest from queue FRenderRequest* nextRenderRequest = nullptr; RenderRequestQueue.Peek(nextRenderRequest); - if (nextRenderRequest && nextRenderRequest->RenderFence.IsFenceComplete()) + if (nextRenderRequest && nextRenderRequest->RenderFence.IsFenceComplete()) { if (CameraType == EROS2CameraType::RGB || CameraType == EROS2CameraType::SEGMENT) { @@ -188,7 +188,7 @@ FROSImg URRROS2CameraComponent::GetROS2Data() else if (CameraType == EROS2CameraType::DEPTH) { // Process Depth data - for (int i = 0; i < nextRenderRequest->Depth.Num(); i++) + for (int i = 0; i < nextRenderRequest->Depth.Num(); i++) { float value = nextRenderRequest->Depth[i].R.GetFloat() / 100; std::memcpy(&Data.Data[i * 4], &value, sizeof(value)); diff --git a/Source/RapyutaSimulationPlugins/Public/Sensors/RRROS2CameraComponent.h b/Source/RapyutaSimulationPlugins/Public/Sensors/RRROS2CameraComponent.h index fc82aa84..b9a1f23d 100644 --- a/Source/RapyutaSimulationPlugins/Public/Sensors/RRROS2CameraComponent.h +++ b/Source/RapyutaSimulationPlugins/Public/Sensors/RRROS2CameraComponent.h @@ -120,10 +120,6 @@ class RAPYUTASIMULATIONPLUGINS_API URRROS2CameraComponent : public URRROS2BaseSe UPROPERTY(EditAnywhere, BlueprintReadWrite) bool Render = true; - // Allow to disable publishing (no image will be published) - UPROPERTY(EditAnywhere, BlueprintReadWrite) - bool Publish = true; - // ROS /** * @brief Update ROS 2 Msg structure from #RenderRequestQueue