diff --git a/controller/src/main/java/com/linbit/linstor/api/rest/v1/ResourceDefinitions.java b/controller/src/main/java/com/linbit/linstor/api/rest/v1/ResourceDefinitions.java index 58a581e32..13e87174d 100644 --- a/controller/src/main/java/com/linbit/linstor/api/rest/v1/ResourceDefinitions.java +++ b/controller/src/main/java/com/linbit/linstor/api/rest/v1/ResourceDefinitions.java @@ -389,7 +389,8 @@ public void clone( requestData.resource_group, requestData.override_props, new HashSet<>(requestData.delete_props), - new HashSet<>(requestData.delete_namespaces) + new HashSet<>(requestData.delete_namespaces), + requestData.rebalance ); requestHelper.doFlux( ApiConsts.API_CLONE_RSCDFN, diff --git a/controller/src/main/java/com/linbit/linstor/api/rest/v1/SnapshotRestoreResource.java b/controller/src/main/java/com/linbit/linstor/api/rest/v1/SnapshotRestoreResource.java index b64ab9cb7..1acb817d7 100644 --- a/controller/src/main/java/com/linbit/linstor/api/rest/v1/SnapshotRestoreResource.java +++ b/controller/src/main/java/com/linbit/linstor/api/rest/v1/SnapshotRestoreResource.java @@ -64,7 +64,8 @@ public void restoreResource( rscName, snapName, snapRestore.to_resource, - snapRestore.stor_pool_rename == null ? Collections.emptyMap() : snapRestore.stor_pool_rename + snapRestore.stor_pool_rename == null ? Collections.emptyMap() : snapRestore.stor_pool_rename, + snapRestore.rebalance ); requestHelper.doFlux( diff --git a/controller/src/main/java/com/linbit/linstor/api/rest/v1/serializer/JsonGenTypes.java b/controller/src/main/java/com/linbit/linstor/api/rest/v1/serializer/JsonGenTypes.java index 4ebdaeeea..c6ede95a9 100644 --- a/controller/src/main/java/com/linbit/linstor/api/rest/v1/serializer/JsonGenTypes.java +++ b/controller/src/main/java/com/linbit/linstor/api/rest/v1/serializer/JsonGenTypes.java @@ -1348,6 +1348,10 @@ public static class SnapshotRestore */ public List nodes = Collections.emptyList(); public Map stor_pool_rename = Collections.emptyMap(); + /** + * If true, after restore completes, replicas will be migrated to the autoplacer-optimal nodes via migrate-disk. + */ + public @Nullable Boolean rebalance; } @JsonInclude(JsonInclude.Include.NON_EMPTY) @@ -1899,6 +1903,10 @@ public static class ResourceDefinitionCloneRequest public Map override_props = Collections.emptyMap(); public List delete_props = Collections.emptyList(); public List delete_namespaces = Collections.emptyList(); + /** + * If true, after cloning completes, replicas will be migrated to the autoplacer-optimal nodes via migrate-disk. + */ + public @Nullable Boolean rebalance; } /** diff --git a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscDfnApiCallHandler.java b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscDfnApiCallHandler.java index 3f0e1d675..68c5f5f7e 100644 --- a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscDfnApiCallHandler.java +++ b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscDfnApiCallHandler.java @@ -167,6 +167,7 @@ public class CtrlRscDfnApiCallHandler private final Provider propsChangeListenerBuilder; private final Autoplacer autoplacer; private final BackgroundRunner backgroundRunner; + private final CtrlRscRebalanceHelper rscRebalanceHelper; @Inject public CtrlRscDfnApiCallHandler( @@ -204,7 +205,8 @@ public CtrlRscDfnApiCallHandler( CtrlRscDfnApiCallHelper ctrlRscDfnApiCallHelperRef, Provider propsChangeListenerBuilderRef, Autoplacer autoplacerRef, - BackgroundRunner backgroundRunnerRef + BackgroundRunner backgroundRunnerRef, + CtrlRscRebalanceHelper rscRebalanceHelperRef ) { errorReporter = errorReporterRef; @@ -242,6 +244,7 @@ public CtrlRscDfnApiCallHandler( propsChangeListenerBuilder = propsChangeListenerBuilderRef; autoplacer = autoplacerRef; backgroundRunner = backgroundRunnerRef; + rscRebalanceHelper = rscRebalanceHelperRef; } public @Nullable ResourceDefinition createResourceDefinition( @@ -820,7 +823,8 @@ public Flux cloneRscDfn( @Nullable String intoRscGrpName, Map overrideProps, Set deletePropKeys, - Set deleteNamespaces + Set deleteNamespaces, + @Nullable Boolean rebalance ) { ResponseContext context = makeResourceDefinitionContext( @@ -860,7 +864,8 @@ public Flux cloneRscDfn( intoRscGrpName, overrideProps, deletePropKeys, - deleteNamespaces + deleteNamespaces, + rebalance ) ) .transform(responses -> responseConverter.reportingExceptions(context, responses)) @@ -1318,7 +1323,8 @@ public Flux cloneRscDfnInTransaction( @Nullable String intoRscGrpName, Map overrideProps, Set deletePropKeys, - Set deleteNamespaces) + Set deleteNamespaces, + @Nullable Boolean rebalance) { Flux flux; @@ -1535,6 +1541,13 @@ public Flux cloneRscDfnInTransaction( .concatWith(resumeIOAndClearCloneProp(srcRscDfn, clonedRscName)) .concatWith(deploymentResponses) .concatWith(removeStartCloning(clonedRscDfn)) + .concatWith(Boolean.TRUE.equals(rebalance) + ? rscRebalanceHelper.trigger(clonedRscDfn) + .onErrorResume(exc -> { + errorReporter.reportError(exc); + return Flux.empty(); + }) + : Flux.empty()) .onErrorResume(exception -> resumeIOAndClearCloneProp(srcRscDfn, clonedRscName)); } catch (ApiRcException exc) diff --git a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscRebalanceHelper.java b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscRebalanceHelper.java new file mode 100644 index 000000000..998c2a729 --- /dev/null +++ b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscRebalanceHelper.java @@ -0,0 +1,208 @@ +package com.linbit.linstor.core.apicallhandler.controller; + +import com.linbit.linstor.annotation.Nullable; +import com.linbit.linstor.annotation.PeerContext; +import com.linbit.linstor.api.ApiCallRc; +import com.linbit.linstor.api.ApiCallRcImpl; +import com.linbit.linstor.api.ApiConsts; +import com.linbit.linstor.api.pojo.AutoSelectFilterPojo; +import com.linbit.linstor.api.pojo.builder.AutoSelectFilterBuilder; +import com.linbit.linstor.core.apicallhandler.controller.autoplacer.Autoplacer; +import com.linbit.linstor.core.apicallhandler.response.ApiAccessDeniedException; +import com.linbit.linstor.core.objects.Node; +import com.linbit.linstor.core.objects.Resource; +import com.linbit.linstor.core.objects.ResourceDefinition; +import com.linbit.linstor.core.objects.StorPool; +import com.linbit.linstor.logging.ErrorReporter; +import com.linbit.linstor.security.AccessContext; +import com.linbit.linstor.security.AccessDeniedException; + +import javax.inject.Inject; +import javax.inject.Provider; +import javax.inject.Singleton; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import reactor.core.publisher.Flux; + +/** + * Helper for rebalancing resources after clone or snapshot restore. + * + * When resources are cloned or restored from snapshot, replicas are placed on the + * same nodes as the source. This helper migrates replicas to the autoplacer-optimal + * nodes via sequential migrate-disk (toggle-disk with MigrateFrom) calls. + */ +@Singleton +public class CtrlRscRebalanceHelper +{ + private final ErrorReporter errorReporter; + private final Autoplacer autoplacer; + private final CtrlRscToggleDiskApiCallHandler rscToggleDiskHelper; + private final Provider peerAccCtx; + + @Inject + public CtrlRscRebalanceHelper( + ErrorReporter errorReporterRef, + Autoplacer autoplacerRef, + CtrlRscToggleDiskApiCallHandler rscToggleDiskHelperRef, + @PeerContext Provider peerAccCtxRef + ) + { + errorReporter = errorReporterRef; + autoplacer = autoplacerRef; + rscToggleDiskHelper = rscToggleDiskHelperRef; + peerAccCtx = peerAccCtxRef; + } + + /** + * Trigger rebalance for the given resource definition. + * + * Computes optimal placement via autoplacer, then sequentially calls + * migrate-disk for each misplaced replica. + * + * @param rscDfn the resource definition to rebalance + * @return Flux of ApiCallRc from the migrate-disk operations + */ + public Flux trigger(ResourceDefinition rscDfn) + { + try + { + AccessContext accCtx = peerAccCtx.get(); + String rscName = rscDfn.getName().displayValue; + + List currentDiskful = rscDfn.getDiskfulResources(accCtx); + if (currentDiskful.isEmpty()) + { + return Flux.empty(); + } + + int replicaCount = currentDiskful.size(); + long rscSize = CtrlRscAutoPlaceApiCallHandler.calculateResourceDefinitionSize(rscDfn, accCtx); + + // Query autoplacer for optimal fresh placement (null rscDfn = ignore existing replicas) + AutoSelectFilterPojo selectFilter = AutoSelectFilterPojo.merge( + new AutoSelectFilterBuilder() + .setPlaceCount(replicaCount) + .build(), + rscDfn.getResourceGroup().getAutoPlaceConfig().getApiData() + ); + + @Nullable Set optimalPools = autoplacer.autoPlace(selectFilter, null, rscSize); + if (optimalPools == null) + { + errorReporter.logWarning( + "Rebalance: autoplacer could not find optimal placement for '%s', skipping rebalance", + rscName + ); + return Flux.empty(); + } + + Set optimalNodeNames = optimalPools.stream() + .map(sp -> sp.getNode().getName().displayValue) + .collect(Collectors.toSet()); + + Set currentNodeNames = currentDiskful.stream() + .map(rsc -> rsc.getNode().getName().displayValue) + .collect(Collectors.toSet()); + + // Compute migration pairs: source nodes that are not optimal -> target nodes that are not current + List nodesToRemove = currentNodeNames.stream() + .filter(n -> !optimalNodeNames.contains(n)) + .collect(Collectors.toList()); + + List nodesToAdd = optimalNodeNames.stream() + .filter(n -> !currentNodeNames.contains(n)) + .collect(Collectors.toList()); + + int pairCount = Math.min(nodesToRemove.size(), nodesToAdd.size()); + if (pairCount == 0) + { + errorReporter.logInfo( + "Rebalance: '%s' is already on optimal nodes, no migration needed", + rscName + ); + return Flux.empty(); + } + + // Build migration pairs: source -> target (with target storage pool name) + Map targetPoolToSource = new LinkedHashMap<>(); + for (int i = 0; i < pairCount; i++) + { + String targetNodeName = nodesToAdd.get(i); + String sourceNodeName = nodesToRemove.get(i); + + // Find the storage pool for target node from autoplacer results + StorPool targetPool = optimalPools.stream() + .filter(sp -> sp.getNode().getName().displayValue.equals(targetNodeName)) + .findFirst() + .orElse(null); + + if (targetPool != null) + { + targetPoolToSource.put(targetPool, sourceNodeName); + } + } + + if (targetPoolToSource.isEmpty()) + { + return Flux.empty(); + } + + List migrationSummary = new ArrayList<>(); + for (Map.Entry entry : targetPoolToSource.entrySet()) + { + migrationSummary.add( + entry.getValue() + " -> " + entry.getKey().getNode().getName().displayValue + ); + } + errorReporter.logInfo( + "Rebalance: '%s' scheduling %d migration(s): %s", + rscName, + pairCount, + String.join(", ", migrationSummary) + ); + + // Chain sequential migrate-disk calls via toggle-disk + Flux migrationFlux = Flux.empty(); + for (Map.Entry entry : targetPoolToSource.entrySet()) + { + StorPool targetPool = entry.getKey(); + String sourceNodeName = entry.getValue(); + String targetNodeName = targetPool.getNode().getName().displayValue; + String storPoolName = targetPool.getName().displayValue; + + migrationFlux = migrationFlux.concatWith( + rscToggleDiskHelper.resourceToggleDisk( + targetNodeName, + rscName, + storPoolName, + sourceNodeName, // migrateFrom + null, // layerList + false, // removeDisk = false (adding disk) + Resource.DiskfulBy.USER + ) + ); + } + + return Flux.just( + ApiCallRcImpl.singleApiCallRc( + ApiConsts.MASK_INFO, + "Rebalance: scheduling " + pairCount + " migration(s) for resource '" + rscName + "'" + ) + ).concatWith(migrationFlux); + } + catch (AccessDeniedException exc) + { + throw new ApiAccessDeniedException( + exc, + "rebalancing resource", + ApiConsts.FAIL_ACC_DENIED_RSC + ); + } + } +} diff --git a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlSnapshotRestoreApiCallHandler.java b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlSnapshotRestoreApiCallHandler.java index 0d62c36bb..6e7f33bee 100644 --- a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlSnapshotRestoreApiCallHandler.java +++ b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlSnapshotRestoreApiCallHandler.java @@ -95,6 +95,7 @@ public class CtrlSnapshotRestoreApiCallHandler private final CtrlPropsHelper ctrlPropsHelper; private final BackupInfoManager backupInfoMgr; private final CtrlSatelliteUpdateCaller ctrlStltUpdateCaller; + private final CtrlRscRebalanceHelper rscRebalanceHelper; @Inject public CtrlSnapshotRestoreApiCallHandler( @@ -112,7 +113,8 @@ public CtrlSnapshotRestoreApiCallHandler( CtrlRscAutoHelper ctrlRscAutoHelperRef, CtrlPropsHelper ctrlPropsHelperRef, BackupInfoManager backupInfoMgrRef, - CtrlSatelliteUpdateCaller ctrlStltUpdateCallerRef + CtrlSatelliteUpdateCaller ctrlStltUpdateCallerRef, + CtrlRscRebalanceHelper rscRebalanceHelperRef ) { errorReporter = errorReporterRef; @@ -130,6 +132,7 @@ public CtrlSnapshotRestoreApiCallHandler( ctrlPropsHelper = ctrlPropsHelperRef; backupInfoMgr = backupInfoMgrRef; ctrlStltUpdateCaller = ctrlStltUpdateCallerRef; + rscRebalanceHelper = rscRebalanceHelperRef; } private ResponseContext makeSnapshotRestoreContext(String rscNameStr) @@ -151,7 +154,8 @@ public Flux restoreSnapshot( String fromRscNameStr, String fromSnapshotNameStr, String toRscNameStr, - Map renameStorPoolMap + Map renameStorPoolMap, + @Nullable Boolean rebalance ) { ResponseContext context = makeSnapshotRestoreContext(toRscNameStr); @@ -163,7 +167,8 @@ public Flux restoreSnapshot( LinstorParsingUtils.asRscName(fromRscNameStr), LinstorParsingUtils.asSnapshotName(fromSnapshotNameStr), LinstorParsingUtils.asRscName(toRscNameStr), - renameStorPoolMap + renameStorPoolMap, + rebalance ).transform(responses -> responseConverter.reportingExceptions(context, responses)); } catch (ApiRcException exc) @@ -178,7 +183,8 @@ public Flux restoreSnapshot( ResourceName fromRscName, SnapshotName fromSnapshotName, ResourceName toRscName, - Map renameStorPoolMap + Map renameStorPoolMap, + @Nullable Boolean rebalance ) { return scopeRunner.fluxInTransactionalScope( @@ -194,7 +200,8 @@ public Flux restoreSnapshot( toRscName, false, true, - renameStorPoolMap + renameStorPoolMap, + rebalance ) ); } @@ -220,7 +227,8 @@ public Flux restoreSnapshotForRollback( toRscName, false, false, - renameStorPoolMap + renameStorPoolMap, + null ) ); } @@ -245,7 +253,8 @@ public Flux restoreSnapshotFromBackup( toRscName, true, false, - Collections.emptyMap() // rename-storpool already happened during download + Collections.emptyMap(), // rename-storpool already happened during download + null ) ).transform(responses -> responseConverter.reportingExceptions(context, responses)); } @@ -257,7 +266,8 @@ private Flux restoreResourceInTransaction( ResourceName toRscName, boolean fromBackup, boolean fromApi, - Map renameStorPoolMap + Map renameStorPoolMap, + @Nullable Boolean rebalance ) { Flux deploymentResponses = Flux.just(); @@ -399,6 +409,13 @@ private Flux restoreResourceInTransaction( return Flux.just(responses) .concatWith(deploymentResponses) .concatWith(autoFlux) + .concatWith(Boolean.TRUE.equals(rebalance) + ? rscRebalanceHelper.trigger(toRscDfn) + .onErrorResume(exc -> { + errorReporter.reportError(exc); + return Flux.empty(); + }) + : Flux.empty()) .concatWith(cleanupFlux) .onErrorResume(CtrlResponseUtils.DelayedApiRcException.class, ignored -> cleanupFlux) .onErrorResume( diff --git a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlSnapshotRollbackApiCallHandler.java b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlSnapshotRollbackApiCallHandler.java index 4df7e41a4..af2b239d5 100644 --- a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlSnapshotRollbackApiCallHandler.java +++ b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlSnapshotRollbackApiCallHandler.java @@ -637,7 +637,8 @@ private Flux rollbackToSafetySnapInTransaction(SnapshotDefinition sna rscName, safetySnapDfn.getName(), rscName, - Collections.emptyMap() + Collections.emptyMap(), + null ); if (updateRscDfn) { diff --git a/docs/rest_v1_openapi.yaml b/docs/rest_v1_openapi.yaml index 366e64ed4..44c90d7f5 100644 --- a/docs/rest_v1_openapi.yaml +++ b/docs/rest_v1_openapi.yaml @@ -7909,6 +7909,11 @@ components: type: object additionalProperties: type: string + rebalance: + type: boolean + description: > + If true, after restore completes, replicas will be migrated + to the autoplacer-optimal nodes via migrate-disk. SnapshotRollback: type: object properties: @@ -8656,6 +8661,11 @@ components: type: array items: type: string + rebalance: + type: boolean + description: > + If true, after cloning completes, replicas will be migrated + to the autoplacer-optimal nodes via migrate-disk. ResourceDefinitionCloneStarted: type: object description: Clone request started object