Merge remote-tracking branch 'upstream/master' into HBASE-29368-feature

Author: haridsv

hbase-archetypes/README.md

@@ -17,9 +17,9 @@ specific language governing permissions and limitations
 under the License.
 -->
 
-#hbase-archetypes
+# hbase-archetypes
 
-##Overview
+## Overview
 The hbase-archetypes subproject of hbase provides an infrastructure for
 creation and maintenance of Maven archetypes<sup id="a1">[1](#f1)</sup>
 pertinent to HBase. Upon deployment to the archetype
@@ -30,9 +30,9 @@ end-user developers to autogenerate completely configured Maven projects
 `archetype:generate` goal of the
 maven-archetype-plugin<sup id="a4">[4](#f4)</sup>.
 
-##Notes for contributors and committers to the HBase project
+## Notes for contributors and committers to the HBase project
 
-####The structure of hbase-archetypes
+#### The structure of hbase-archetypes
 The hbase-archetypes project contains a separate subproject for each archetype.
 The top level components of such a subproject comprise a complete, standalone
 exemplar Maven project containing:
@@ -47,7 +47,7 @@ code `./src/main/.../HelloHBase.java` and `./src/test/.../TestHelloHBase.java`,
 (b) a `pom.xml` file establishing dependency upon hbase-client and test-scope
 dependency upon hbase-testing-util, and (c) a `log4j.properties` resource file.
 
-####How archetypes are created during the hbase install process
+#### How archetypes are created during the hbase install process
 During the `mvn install` process, all standalone exemplar projects in the
 `hbase-archetypes` subdirectory are first packaged/tested/installed, and then
 the following steps are executed in `hbase-archetypes/hbase-archetype-builder`
@@ -74,7 +74,7 @@ repository. (Note that installation of an archetype automatically includes
 invocation of integration-testing prior to install, which performs a test
 generation of a project from the archetype.)
 
-####How to add a new archetype to the hbase-archetypes collection
+#### How to add a new archetype to the hbase-archetypes collection
 1. Create a new subdirectory in `hbase-archetypes`, populated with a
 completely configured Maven project, which will serve as the exemplar project
 of the new archetype. (It may be most straightforward to simply copy the `src`
@@ -93,7 +93,7 @@ elements and `<transformationSet>` elements within the `<plugin>` elements
 `createArchetypes.sh` and `installArchetypes.sh` scripts in the
 `hbase-archetype-builder` subdirectory (using the existing entries as a guide).
 
-####How to do additional testing/inspection of an archetype in this collection
+#### How to do additional testing/inspection of an archetype in this collection
 Although integration-testing (which is automatically performed for each
 archetype during the install process) already performs test generation of a
 project from an archetype, it may often be advisable to do further manual
@@ -106,7 +106,7 @@ and can be tested locally by executing the following:
 This displays a numbered list of all locally-installed archetypes for the user
 to choose from for generation of a new Maven project.
 
-##Footnotes:
+## Footnotes:
 <b id="f1">1</b> -- [Maven Archetype
 ](http://maven.apache.org/archetype/index.html) ("About" page).
 -- [↩](#a1)

hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java

@@ -554,6 +554,10 @@ public Scan setMaxResultSize(long maxResultSize) {
 
   @Override
   public Scan setFilter(Filter filter) {
+    if (filter != null && filter.hasFilterRow() && this.batch > 0) {
+      throw new IncompatibleFilterException(
+        "Cannot set a filter that returns true for filter.hasFilterRow on a scan with batch set");
+    }
     super.setFilter(filter);
     return this;
   }

hbase-client/src/main/java/org/apache/hadoop/hbase/filter/MultipleColumnPrefixFilter.java

@@ -37,9 +37,10 @@
 import org.apache.hadoop.hbase.shaded.protobuf.generated.FilterProtos;
 
 /**
- * This filter is used for selecting only those keys with columns that matches a particular prefix.
- * For example, if prefix is 'an', it will pass keys will columns like 'and', 'anti' but not keys
- * with columns like 'ball', 'act'.
+ * This filter is used for selecting only those keys with columns that match any of the given
+ * prefixes. For example, if prefixes are 'an' and 'ba', it will pass keys with columns like 'and',
+ * 'anti', 'ball' but not keys with columns like 'cat', 'act'. The prefixes are stored in a sorted
+ * set and the filter uses seek hints to efficiently skip columns that do not match any prefix.
  */
 @InterfaceAudience.Public
 public class MultipleColumnPrefixFilter extends FilterBase implements HintingFilter {

hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOperation.java

@@ -371,7 +371,7 @@ public void testScanOperationToJSON() throws IOException {
     scan.setLimit(5);
     scan.setReadType(Scan.ReadType.PREAD);
     scan.setNeedCursorResult(true);
-    scan.setFilter(SCV_FILTER);
+    scan.setFilter(VALUE_FILTER);
     scan.setReplicaId(1);
     scan.setConsistency(Consistency.STRONG);
     scan.setLoadColumnFamiliesOnDemand(true);

hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestScan.java

@@ -19,6 +19,7 @@
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 
@@ -29,6 +30,8 @@
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.client.Scan.ReadType;
 import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.IncompatibleFilterException;
+import org.apache.hadoop.hbase.filter.PageFilter;
 import org.apache.hadoop.hbase.security.access.Permission;
 import org.apache.hadoop.hbase.security.visibility.Authorizations;
 import org.apache.hadoop.hbase.testclassification.ClientTests;
@@ -254,6 +257,36 @@ public void testScanCopyConstructor() throws Exception {
       "Make sure copy constructor adds all the fields in the copied object");
   }
 
+  @Test
+  public void testSetFilterWithBatchThrows() {
+    Scan scan = new Scan();
+    scan.setBatch(5);
+    assertThrows(IncompatibleFilterException.class, () -> scan.setFilter(new PageFilter(10)));
+  }
+
+  @Test
+  public void testSetFilterWithoutBatchDoesNotThrow() {
+    Scan scan = new Scan();
+    scan.setFilter(new PageFilter(10));
+    // no exception expected
+  }
+
+  @Test
+  public void testSetFilterWithBatchAndNonFilterRowFilter() {
+    Scan scan = new Scan();
+    scan.setBatch(5);
+    scan.setFilter(new FilterList());
+    // FilterList.hasFilterRow() returns false, so no exception expected
+  }
+
+  @Test
+  public void testSetFilterWithBatchAndNullFilter() {
+    Scan scan = new Scan();
+    scan.setBatch(5);
+    scan.setFilter(null);
+    // null filter should not throw
+  }
+
   @Test
   public void testScanReadType() throws Exception {
     Scan scan = new Scan();

hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCompactionChore.java

@@ -102,15 +102,15 @@ protected void chore() {
                 hcd.getNameAsString());
               if (regionBatchSize == MobConstants.DEFAULT_MOB_MAJOR_COMPACTION_REGION_BATCH_SIZE) {
                 LOG.debug(
-                  "Table={} cf ={}: batch MOB compaction is disabled, {}=0 -"
+                  "Table={} cf={}: batch MOB compaction is disabled, {}=0 -"
                     + " all regions will be compacted in parallel",
-                  htd.getTableName(), hcd.getNameAsString(), "hbase.mob.compaction.batch.size");
+                  htd.getTableName(), hcd.getNameAsString(),
+                  MobConstants.MOB_MAJOR_COMPACTION_REGION_BATCH_SIZE);
                 admin.majorCompact(htd.getTableName(), hcd.getName());
               } else {
-                LOG.info(
-                  "Table={} cf={}: performing MOB major compaction in batches "
-                    + "'hbase.mob.compaction.batch.size'={}",
-                  htd.getTableName(), hcd.getNameAsString(), regionBatchSize);
+                LOG.info("Table={} cf={}: performing MOB major compaction in batches {}={}",
+                  htd.getTableName(), hcd.getNameAsString(),
+                  MobConstants.MOB_MAJOR_COMPACTION_REGION_BATCH_SIZE, regionBatchSize);
                 performMajorCompactionInBatches(admin, htd, hcd);
               }
             } else {

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/DeleteTracker.java

@@ -83,6 +83,20 @@ enum DeleteResult {
                    // deleted in strong semantics of versions(See MvccTracker)
   }
 
+  /**
+   * Check if the given delete marker is redundant, i.e., it is already covered by a previously
+   * tracked delete of equal or broader scope. A DeleteFamily is redundant if a DeleteFamily with a
+   * higher timestamp was already seen. A DeleteColumn is redundant if a DeleteColumn for the same
+   * qualifier with a higher timestamp, or a DeleteFamily with a higher timestamp, was already seen.
+   * <p>
+   * This is a read-only check with no side effects on tracker state.
+   * @param cell the delete marker cell to check
+   * @return true if the delete marker is redundant and can be skipped
+   */
+  default boolean isRedundantDelete(ExtendedCell cell) {
+    return false;
+  }
+
   /**
    * Return the comparator passed to this delete tracker
    * @return the cell comparator

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/MinorCompactionScanQueryMatcher.java

@@ -19,6 +19,7 @@
 
 import java.io.IOException;
 import org.apache.hadoop.hbase.ExtendedCell;
+import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.PrivateCellUtil;
 import org.apache.hadoop.hbase.regionserver.ScanInfo;
 import org.apache.yetus.audience.InterfaceAudience;
@@ -47,6 +48,19 @@ public MatchCode match(ExtendedCell cell) throws IOException {
         // we should not use this delete marker to mask any cell yet.
         return MatchCode.INCLUDE;
       }
+      // Check before tracking: an older DeleteColumn or DeleteFamily is redundant if a newer
+      // one of equal or broader scope was already seen. Must check before trackDelete() since
+      // that overwrites tracker state. Seek past remaining cells for this column/row since
+      // they are all covered by the previously tracked delete.
+      if (deletes.isRedundantDelete(cell)) {
+        // Skip seeking for deletes with empty qualifier, not to skip a subsequent
+        // DeleteFamily marker that covers other qualifiers. DeleteFamily itself can seek
+        // safely because all remaining empty-qualifier cells are redundant under it.
+        if (cell.getQualifierLength() == 0 && typeByte != KeyValue.Type.DeleteFamily.getCode()) {
+          return MatchCode.SKIP;
+        }
+        return columns.getNextRowOrNextColumn(cell);
+      }
       trackDelete(cell);
       return MatchCode.INCLUDE;
     }

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/NormalUserScanQueryMatcher.java

@@ -18,8 +18,11 @@
 package org.apache.hadoop.hbase.regionserver.querymatcher;
 
 import java.io.IOException;
+import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.ExtendedCell;
 import org.apache.hadoop.hbase.KeepDeletedCells;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueUtil;
 import org.apache.hadoop.hbase.PrivateCellUtil;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.regionserver.ScanInfo;
@@ -31,6 +34,14 @@
 @InterfaceAudience.Private
 public abstract class NormalUserScanQueryMatcher extends UserScanQueryMatcher {
 
+  /**
+   * Number of consecutive range delete markers (DeleteColumn/DeleteFamily) to skip before switching
+   * to seek. Seeking is more expensive than skipping for a single marker, but much faster when
+   * markers accumulate. This threshold avoids the seek overhead for the common case (one delete per
+   * row/column) while still kicking in when markers pile up.
+   */
+  static final int SEEK_ON_DELETE_MARKER_THRESHOLD = 10;
+
   /** Keeps track of deletes */
   private final DeleteTracker deletes;
 
@@ -40,18 +51,32 @@ public abstract class NormalUserScanQueryMatcher extends UserScanQueryMatcher {
   /** whether time range queries can see rows "behind" a delete */
   protected final boolean seePastDeleteMarkers;
 
+  /** Whether seek optimization for range delete markers is applicable */
+  private final boolean canSeekOnDeleteMarker;
+
+  /** Count of consecutive range delete markers seen for the same column */
+  private int rangeDeleteCount;
+
+  /** Last range delete cell, for qualifier comparison across consecutive markers */
+  private ExtendedCell lastDelete;
+
   protected NormalUserScanQueryMatcher(Scan scan, ScanInfo scanInfo, ColumnTracker columns,
     boolean hasNullColumn, DeleteTracker deletes, long oldestUnexpiredTS, long now) {
     super(scan, scanInfo, columns, hasNullColumn, oldestUnexpiredTS, now);
     this.deletes = deletes;
     this.get = scan.isGetScan();
     this.seePastDeleteMarkers = scanInfo.getKeepDeletedCells() != KeepDeletedCells.FALSE;
+    this.canSeekOnDeleteMarker =
+      !seePastDeleteMarkers && deletes.getClass() == ScanDeleteTracker.class;
   }
 
   @Override
   public void beforeShipped() throws IOException {
     super.beforeShipped();
     deletes.beforeShipped();
+    if (lastDelete != null) {
+      lastDelete = KeyValueUtil.toNewKeyCell(lastDelete);
+    }
   }
 
   @Override
@@ -71,8 +96,31 @@ public MatchCode match(ExtendedCell cell) throws IOException {
       if (includeDeleteMarker) {
         this.deletes.add(cell);
       }
+
+      // A DeleteColumn or DeleteFamily masks all remaining cells for this column/family.
+      // Seek past them instead of skipping one cell at a time, but only after seeing
+      // enough consecutive markers for the same column to justify the seek overhead.
+      // Only safe with plain ScanDeleteTracker. Not safe with newVersionBehavior (sequence
+      // IDs determine visibility), visibility labels (delete/put label mismatch), or
+      // seePastDeleteMarkers (KEEP_DELETED_CELLS).
+      if (
+        canSeekOnDeleteMarker && (typeByte == KeyValue.Type.DeleteFamily.getCode()
+          || (typeByte == KeyValue.Type.DeleteColumn.getCode() && cell.getQualifierLength() > 0))
+      ) {
+        if (lastDelete != null && !CellUtil.matchingQualifier(cell, lastDelete)) {
+          rangeDeleteCount = 0;
+        }
+        lastDelete = cell;
+        if (++rangeDeleteCount >= SEEK_ON_DELETE_MARKER_THRESHOLD) {
+          rangeDeleteCount = 0;
+          return columns.getNextRowOrNextColumn(cell);
+        }
+      } else {
+        rangeDeleteCount = 0;
+      }
       return MatchCode.SKIP;
     }
+    rangeDeleteCount = 0;
     returnCode = checkDeleted(deletes, cell);
     if (returnCode != null) {
       return returnCode;
@@ -83,6 +131,8 @@ public MatchCode match(ExtendedCell cell) throws IOException {
   @Override
   protected void reset() {
     deletes.reset();
+    rangeDeleteCount = 0;
+    lastDelete = null;
   }
 
   @Override

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/ScanDeleteTracker.java

@@ -142,6 +142,28 @@ public DeleteResult isDeleted(ExtendedCell cell) {
     return DeleteResult.NOT_DELETED;
   }
 
+  @Override
+  public boolean isRedundantDelete(ExtendedCell cell) {
+    byte type = cell.getTypeByte();
+    boolean coveredByFamily = hasFamilyStamp && cell.getTimestamp() <= familyStamp;
+
+    if (
+      type == KeyValue.Type.DeleteFamily.getCode()
+        || type == KeyValue.Type.DeleteFamilyVersion.getCode()
+    ) {
+      return coveredByFamily;
+    }
+
+    boolean coveredByColumn =
+      deleteCell != null && deleteType == KeyValue.Type.DeleteColumn.getCode()
+        && CellUtil.matchingQualifier(cell, deleteCell) && cell.getTimestamp() <= deleteTimestamp;
+
+    if (type == KeyValue.Type.DeleteColumn.getCode() || type == KeyValue.Type.Delete.getCode()) {
+      return coveredByFamily || coveredByColumn;
+    }
+    return false;
+  }
+
   @Override
   public boolean isEmpty() {
     return deleteCell == null && !hasFamilyStamp && familyVersionStamps.isEmpty();