Add support for changing deletePctAllowed

clientlib/src/main/proto/yelp/nrtsearch/luceneserver.proto

@@ -458,6 +458,8 @@ message LiveSettingsRequest {
     int32 defaultTerminateAfter = 14;
     //Terminate after max recall count value to use when not specified in the search request.
     int32 defaultTerminateAfterMaxRecallCount = 15;
+    // Controls the maximum percentage of deleted documents that is tolerated in the index
+    double deletePctAllowed = 16;
 }
 
 // Response from Server to liveSettings
@@ -1469,6 +1471,8 @@ message IndexLiveSettings {
     google.protobuf.Int32Value parallelFetchChunkSize = 17;
     // Terminate after max recall count value to use when not specified in the search request, or 0 for none, default: 0
     google.protobuf.Int32Value defaultTerminateAfterMaxRecallCount = 18;
+    // Controls the maximum percentage of deleted documents that is tolerated in the index, default: 20
+    google.protobuf.DoubleValue deletePctAllowed = 19;
 }
 
 // Index state

grpc-gateway/luceneserver.swagger.json

@@ -1394,6 +1394,14 @@
             "type": "integer",
             "format": "int32"
           },
+          {
+            "name": "liveSettings.deletePctAllowed",
+            "description": "Controls the maximum percentage of deleted documents that is tolerated in the index, default: 20",
+            "in": "query",
+            "required": false,
+            "type": "number",
+            "format": "double"
+          },
           {
             "name": "local",
             "description": "When set to true, live settings changes are only applied to the local node. These changes are ephemeral,\nso will not persist through a restart. Also, the live settings returned in the response will contain the\nlocal overrides only when this flag is true.",
@@ -2458,7 +2466,7 @@
       "properties": {
         "indexName": {
           "type": "string",
-          "title": "Index name"
+          "description": "Index name; use this parameter when you want to add documents to one specific index.\nYou must exclusively use this parameter or the indexNames parameter, not both."
         },
         "fields": {
           "type": "object",
@@ -2470,6 +2478,13 @@
         "requestType": {
           "$ref": "#/definitions/luceneserverIndexingRequestType",
           "title": "request type"
+        },
+        "indexNames": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "description": "List of indexNames; use this parameter when you want to add documents to multiple indices at once.\nYou must exclusively use this parameter or the indexName parameter, not both."
         }
       },
       "title": "Message representing a document to index"
@@ -3964,6 +3979,11 @@
           "type": "integer",
           "format": "int32",
           "title": "Terminate after max recall count value to use when not specified in the search request, or 0 for none, default: 0"
+        },
+        "deletePctAllowed": {
+          "type": "number",
+          "format": "double",
+          "title": "Controls the maximum percentage of deleted documents that is tolerated in the index, default: 20"
         }
       },
       "title": "Index live settings"
@@ -4334,6 +4354,11 @@
           "type": "integer",
           "format": "int32",
           "description": "Terminate after max recall count value to use when not specified in the search request."
+        },
+        "deletePctAllowed": {
+          "type": "number",
+          "format": "double",
+          "title": "Controls the maximum percentage of deleted documents that is tolerated in the index"
         }
       },
       "title": "Input to liveSettings"

src/main/java/com/yelp/nrtsearch/server/handler/LiveSettingsHandler.java

@@ -115,6 +115,10 @@ private LiveSettingsResponse handleAsLiveSettingsV2(
         settingsBuilder.setSegmentsPerTier(
             Int32Value.newBuilder().setValue(liveSettingsRequest.getSegmentsPerTier()).build());
       }
+      if (liveSettingsRequest.getDeletePctAllowed() != 0) {
+        settingsBuilder.setDeletePctAllowed(
+            DoubleValue.newBuilder().setValue(liveSettingsRequest.getDeletePctAllowed()).build());
+      }
       if (liveSettingsRequest.getDefaultSearchTimeoutSec() >= 0) {
         settingsBuilder.setDefaultSearchTimeoutSec(
             DoubleValue.newBuilder()

src/main/java/com/yelp/nrtsearch/server/index/ImmutableIndexState.java

@@ -119,6 +119,7 @@ public class ImmutableIndexState extends IndexState {
   public static final int DEFAULT_SLICE_MAX_SEGMENTS = 5;
   public static final int DEFAULT_VIRTUAL_SHARDS = 1;
   public static final int DEFAULT_SEGMENTS_PER_TIER = 10;
+  public static final double DEFAULT_DELETE_PCT_ALLOWED = 20.0;
   public static final int DEFAULT_MAX_MERGED_SEGMENT_MB = 5 * 1024;
   public static final int DEFAULT_PARALLEL_FETCH_CHUNK_SIZE = 50;
 
@@ -136,6 +137,8 @@ public class ImmutableIndexState extends IndexState {
           .setSliceMaxSegments(Int32Value.newBuilder().setValue(DEFAULT_SLICE_MAX_SEGMENTS).build())
           .setVirtualShards(Int32Value.newBuilder().setValue(DEFAULT_VIRTUAL_SHARDS).build())
           .setSegmentsPerTier(Int32Value.newBuilder().setValue(DEFAULT_SEGMENTS_PER_TIER).build())
+          .setDeletePctAllowed(
+              DoubleValue.newBuilder().setValue(DEFAULT_DELETE_PCT_ALLOWED).build())
           .setMaxMergedSegmentMB(
               Int32Value.newBuilder().setValue(DEFAULT_MAX_MERGED_SEGMENT_MB).build())
           // default unset
@@ -161,6 +164,7 @@ public class ImmutableIndexState extends IndexState {
   private final int virtualShards;
   private final int maxMergedSegmentMB;
   private final int segmentsPerTier;
+  private final double deletePctAllowed;
   private final double defaultSearchTimeoutSec;
   private final int defaultSearchTimeoutCheckEvery;
   private final int defaultTerminateAfter;
@@ -257,6 +261,7 @@ public ImmutableIndexState(
     virtualShards = mergedLiveSettingsWithLocal.getVirtualShards().getValue();
     maxMergedSegmentMB = mergedLiveSettingsWithLocal.getMaxMergedSegmentMB().getValue();
     segmentsPerTier = mergedLiveSettingsWithLocal.getSegmentsPerTier().getValue();
+    deletePctAllowed = mergedLiveSettingsWithLocal.getDeletePctAllowed().getValue();
     defaultSearchTimeoutSec = mergedLiveSettingsWithLocal.getDefaultSearchTimeoutSec().getValue();
     defaultSearchTimeoutCheckEvery =
         mergedLiveSettingsWithLocal.getDefaultSearchTimeoutCheckEvery().getValue();
@@ -608,6 +613,7 @@ public synchronized MergeThread getMergeThread(
     }
     mergePolicy.setMaxMergedSegmentMB(maxMergedSegmentMB);
     mergePolicy.setSegmentsPerTier(segmentsPerTier);
+    mergePolicy.setDeletesPctAllowed(deletePctAllowed);
     iwc.setMergePolicy(mergePolicy);
 
     return iwc;
@@ -712,6 +718,11 @@ public int getSegmentsPerTier() {
     return segmentsPerTier;
   }
 
+  @Override
+  public double getDeletePctAllowed() {
+    return deletePctAllowed;
+  }
+
   @Override
   public double getDefaultSearchTimeoutSec() {
     return defaultSearchTimeoutSec;
@@ -817,6 +828,10 @@ static void validateLiveSettings(IndexLiveSettings liveSettings) {
     if (liveSettings.getSegmentsPerTier().getValue() < 2) {
       throw new IllegalArgumentException("segmentsPerTier must be >= 2");
     }
+    if (liveSettings.getDeletePctAllowed().getValue() < 5.0
+        || liveSettings.getDeletePctAllowed().getValue() > 50.0) {
+      throw new IllegalArgumentException("deletePctAllowed must be between 5.0 and 50.0");
+    }
     if (liveSettings.getDefaultSearchTimeoutSec().getValue() < 0.0) {
       throw new IllegalArgumentException("defaultSearchTimeoutSec must be >= 0.0");
     }

src/main/java/com/yelp/nrtsearch/server/index/IndexState.java

@@ -514,6 +514,9 @@ public abstract IndexWriterConfig getIndexWriterConfig(
   /** Get the number of segments per tier used by merge policy, or 0 if using policy default. */
   public abstract int getSegmentsPerTier();
 
+  /** Get the maximum percentage of deleted documents that is tolerated in the index. */
+  public abstract double getDeletePctAllowed();
+
   /** Get the default search timeout. */
   public abstract double getDefaultSearchTimeoutSec();
 

src/main/java/com/yelp/nrtsearch/tools/cli/LiveSettingsV2Command.java

@@ -91,6 +91,11 @@ public class LiveSettingsV2Command implements Callable<Integer> {
       description = "Number of segments per tier used by TieredMergePolicy")
   private Integer segmentsPerTier;
 
+  @CommandLine.Option(
+      names = {"--deletePctAllowed"},
+      description = "Maximum percentage of deleted documents that is tolerated in the index")
+  private Double deletePctAllowed;
+
   @CommandLine.Option(
       names = {"--defaultSearchTimeoutSec"},
       description = "Search timeout to use when not provided by the request")
@@ -185,6 +190,10 @@ public Integer call() throws Exception {
         liveSettingsBuilder.setSegmentsPerTier(
             Int32Value.newBuilder().setValue(segmentsPerTier).build());
       }
+      if (deletePctAllowed != null) {
+        liveSettingsBuilder.setDeletePctAllowed(
+            DoubleValue.newBuilder().setValue(deletePctAllowed).build());
+      }
       if (defaultSearchTimeoutSec != null) {
         liveSettingsBuilder.setDefaultSearchTimeoutSec(
             DoubleValue.newBuilder().setValue(defaultSearchTimeoutSec).build());

src/test/java/com/yelp/nrtsearch/server/grpc/StateBackendServerTest.java

@@ -1652,6 +1652,7 @@ public void testLiveSettingsV1All() throws IOException {
             .setDefaultSearchTimeoutCheckEvery(500)
             .setDefaultTerminateAfter(5000)
             .setDefaultTerminateAfterMaxRecallCount(6000)
+            .setDeletePctAllowed(20.0)
             .build();
 
     LiveSettingsResponse response = primaryClient.getBlockingStub().liveSettings(request);
@@ -1675,6 +1676,7 @@ public void testLiveSettingsV1All() throws IOException {
             .setVerboseMetrics(BoolValue.newBuilder().setValue(false).build())
             .setParallelFetchByField(BoolValue.newBuilder().setValue(false).build())
             .setParallelFetchChunkSize(Int32Value.newBuilder().setValue(50).build())
+            .setDeletePctAllowed(DoubleValue.newBuilder().setValue(20.0).build())
             .build();
 
     IndexLiveSettings.Builder builder = IndexLiveSettings.newBuilder();

src/test/java/com/yelp/nrtsearch/server/index/ImmutableIndexStateTest.java

@@ -734,6 +734,28 @@ public void testSegmentsPerTier_invalid() throws IOException {
     assertLiveSettingException(expectedMsg, b -> b.setSegmentsPerTier(wrap(1)));
   }
 
+  @Test
+  public void testDeletePctAllowed_default() throws IOException {
+    assertEquals(
+        ImmutableIndexState.DEFAULT_DELETE_PCT_ALLOWED,
+        getIndexState(getEmptyState()).getDeletePctAllowed(),
+        0.0);
+  }
+
+  @Test
+  public void testDeletePctAllowed_set() throws IOException {
+    verifyDoubleLiveSetting(
+        50.0, ImmutableIndexState::getDeletePctAllowed, b -> b.setDeletePctAllowed(wrap(50.0)));
+    verifyDoubleLiveSetting(
+        25.0, ImmutableIndexState::getDeletePctAllowed, b -> b.setDeletePctAllowed(wrap(25.0)));
+  }
+
+  @Test
+  public void testDeletePctAllowed_invalid() throws IOException {
+    String expectedMsg = "deletePctAllowed must be between 5.0 and 50.0";
+    assertLiveSettingException(expectedMsg, b -> b.setDeletePctAllowed(wrap(1.0)));
+  }
+
   @Test
   public void testDefaultSearchTimeoutSec_default() throws IOException {
     assertEquals(0.0, getIndexState(getEmptyState()).getDefaultSearchTimeoutSec(), 0.0);