Add usage of configurable property to remove types when processing unfielded terms

jack-gitdev · jack-gitdev · commit ac4945d40d86 · 2025-08-14T14:07:48.000Z
diff --git a/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java b/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java
@@ -409,6 +409,8 @@ public class ShardQueryConfiguration extends GenericQueryConfiguration implement
 
     private List<String> contentFieldNames = Collections.emptyList();
 
+    private List<Type<?>> excludeUnfieldedTypes = Collections.emptyList();
+
     /**
      * The source to use as the active query log name for all query iterators in scans generated for the shard query logic. If the value
      * {@value #TABLE_NAME_SOURCE} is supplied, the shard table name will be used. If {@value #QUERY_LOGIC_NAME_SOURCE} is supplied, the name of the shard query
@@ -736,6 +738,7 @@ public void copyFrom(ShardQueryConfiguration other) {
         this.setIvaratorCacheBufferSize(other.getIvaratorCacheBufferSize());
         this.setIvaratorCacheScanPersistThreshold(other.getIvaratorCacheScanPersistThreshold());
         this.setIvaratorCacheScanTimeout(other.getIvaratorCacheScanTimeout());
+        this.setExcludeUnfieldedTypes(other.getExcludeUnfieldedTypes());
         this.setMaxFieldIndexRangeSplit(other.getMaxFieldIndexRangeSplit());
         this.setIvaratorMaxOpenFiles(other.getIvaratorMaxOpenFiles());
         this.setIvaratorNumRetries(other.getIvaratorNumRetries());
@@ -1623,6 +1626,14 @@ public void setIvaratorCacheScanTimeout(long ivaratorCacheScanTimeout) {
         this.ivaratorCacheScanTimeout = ivaratorCacheScanTimeout;
     }
 
+    public List<Type<?>> getExcludeUnfieldedTypes() {
+        return excludeUnfieldedTypes;
+    }
+
+    public void setExcludeUnfieldedTypes(List<Type<?>> excludeUnfieldedTypes) {
+        this.excludeUnfieldedTypes = excludeUnfieldedTypes;
+    }
+
     public int getMaxFieldIndexRangeSplit() {
         return maxFieldIndexRangeSplit;
     }
@@ -3021,6 +3032,7 @@ public boolean equals(Object o) {
                 getIvaratorCacheBufferSize() == that.getIvaratorCacheBufferSize() &&
                 getIvaratorCacheScanPersistThreshold() == that.getIvaratorCacheScanPersistThreshold() &&
                 getIvaratorCacheScanTimeout() == that.getIvaratorCacheScanTimeout() &&
+                getExcludeUnfieldedTypes() == that.getExcludeUnfieldedTypes() &&
                 getMaxFieldIndexRangeSplit() == that.getMaxFieldIndexRangeSplit() &&
                 getIvaratorMaxOpenFiles() == that.getIvaratorMaxOpenFiles() &&
                 getIvaratorNumRetries() == that.getIvaratorNumRetries() &&
@@ -3291,6 +3303,7 @@ public int hashCode() {
                 getIvaratorCacheBufferSize(),
                 getIvaratorCacheScanPersistThreshold(),
                 getIvaratorCacheScanTimeout(),
+                getExcludeUnfieldedTypes(),
                 getMaxFieldIndexRangeSplit(),
                 getIvaratorMaxOpenFiles(),
                 getIvaratorNumRetries(),
diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/ExpandMultiNormalizedTerms.java b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/ExpandMultiNormalizedTerms.java
@@ -332,6 +332,10 @@ protected JexlNode expandNodeForNormalizers(JexlNode node, Object data) {
             if (fieldName.equals(Constants.ANY_FIELD)) {
                 try {
                     dataTypes.addAll(helper.getAllDatatypes());
+                    for (Type<?> type : config.getExcludeUnfieldedTypes()) {
+                        dataTypes.removeIf(dataType -> dataType.getClass().equals(type.getClass()));
+                    }
+
                 } catch (InstantiationException | IllegalAccessException | TableNotFoundException e) {
                     log.error("Could not fetch all DataTypes while expanding unfielded term");
                     throw new RuntimeException(e);
diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java
@@ -2412,6 +2412,14 @@ public void setIvaratorCacheScanTimeoutMinutes(long hdfsCacheScanTimeoutMinutes)
         getConfig().setIvaratorCacheScanTimeout(hdfsCacheScanTimeoutMinutes * 1000 * 60);
     }
 
+    public List<Type<?>> getExcludeUnfieldedTypes() {
+        return getConfig().getExcludeUnfieldedTypes();
+    }
+
+    public void setExcludeUnfieldedTypes(List<Type<?>> excludeUnfieldedTypes) {
+        getConfig().setExcludeUnfieldedTypes(excludeUnfieldedTypes);
+    }
+
     public String getHdfsSiteConfigURLs() {
         return getConfig().getHdfsSiteConfigURLs();
     }
diff --git a/warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java b/warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java
@@ -34,6 +34,7 @@
 import datawave.data.type.GeometryType;
 import datawave.data.type.LcNoDiacriticsType;
 import datawave.data.type.NoOpType;
+import datawave.data.type.NumberType;
 import datawave.data.type.Type;
 import datawave.microservice.query.Query;
 import datawave.microservice.query.QueryImpl;
@@ -411,6 +412,8 @@ public void setUp() throws Exception {
         updatedValues.put("ivaratorCacheScanPersistThreshold", 1040L);
         defaultValues.put("ivaratorCacheScanTimeout", 3600000L);
         updatedValues.put("ivaratorCacheScanTimeout", 3600L);
+        defaultValues.put("excludeUnfieldedTypes", Collections.emptyList());
+        updatedValues.put("excludeUnfieldedTypes", Lists.newArrayList(new NumberType()));
         defaultValues.put("maxFieldIndexRangeSplit", 11);
         updatedValues.put("maxFieldIndexRangeSplit", 20);
         defaultValues.put("ivaratorMaxOpenFiles", 100);
diff --git a/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/ExecutableExpansionVisitorTest.java b/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/ExecutableExpansionVisitorTest.java
@@ -6,6 +6,7 @@
 import java.text.DateFormat;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;
@@ -439,14 +440,14 @@ public void testAnyfieldNumericExpansion() throws Exception {
             log.debug("testAnyfieldNumericExpansion");
         }
         String[] queryStrings = {"_ANYFIELD_ =~'12340.*?'"};
-        @SuppressWarnings("unchecked")
-        // SOPRANO is the only one with a 0 after the 1234
-        List<String>[] expectedLists = new List[] {Arrays.asList("SOPRANO")};
+        // ANYFIELD numeric regex normalization is no longer used, so expect no results for this query
+        List<String> expectedLists = new ArrayList<>();
         for (int i = 0; i < queryStrings.length; i++) {
-            runTestQuery(expectedLists[i], queryStrings[i], format.parse("20091231"), format.parse("20150101"), extraParameters);
+            runTestQuery(expectedLists, queryStrings[i], format.parse("20091231"), format.parse("20150101"), extraParameters);
         }
 
-        String expectedQueryStr = "(BAIL == '+eE1.2345' || BAIL == '+fE1.23401') && ((_Eval_ = true) && (_ANYFIELD_ =~ '12340.*?'))";
+        // No longer expect normalized numeric regexes for ANYFIELD
+        String expectedQueryStr = "_NOFIELD_ =~ '12340.*?'";
         String plan = JexlFormattedStringBuildingVisitor.buildQuery(logic.getConfig().getQueryTree());
         Assert.assertTrue("Expected equality: " + expectedQueryStr + " vs " + plan,
                         TreeEqualityVisitor.isEqual(JexlASTHelper.parseJexlQuery(expectedQueryStr), logic.getConfig().getQueryTree()));
diff --git a/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/ExpandMultiNormalizedTermsTest.java b/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/ExpandMultiNormalizedTermsTest.java
@@ -9,6 +9,7 @@
 import static datawave.query.jexl.nodes.QueryPropertyMarker.MarkerType.LENIENT;
 import static datawave.query.jexl.nodes.QueryPropertyMarker.MarkerType.STRICT;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;
@@ -746,6 +747,48 @@ public void testAnyFieldTerms() throws ParseException {
         expandTerms("_ANYFIELD_ >= '123'", "_ANYFIELD_ >= '+cE1.23' || _ANYFIELD_ >= '123'");
     }
 
+    /**
+     * Test type exclusion for each node type
+     *
+     * @throws ParseException
+     *             if the query fails to parse
+     */
+    @Test
+    public void testAnyFieldTermsTypeExclusion() throws ParseException {
+
+        Multimap<String,Type<?>> dataTypes = HashMultimap.create();
+        dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType(), new LcType(), new NumberType(), new NoOpType()));
+        helper.setDataTypes(dataTypes);
+
+        List<Type<?>> excludeUnfieldedTypes = new ArrayList<>();
+        excludeUnfieldedTypes.add(new NumberType());
+        config.setExcludeUnfieldedTypes(excludeUnfieldedTypes);
+
+        // EQ
+        expandTerms("_ANYFIELD_ == '123'", "_ANYFIELD_ == '123'");
+
+        // NE
+        expandTerms("_ANYFIELD_ != '123'", "_ANYFIELD_ != '123'");
+
+        // ER
+        expandTerms("_ANYFIELD_ =~ '123'", "_ANYFIELD_ =~ '123'");
+
+        // NR
+        expandTerms("_ANYFIELD_ !~ '123'", "_ANYFIELD_ !~ '123'");
+
+        // LT
+        expandTerms("_ANYFIELD_ < '123'", "_ANYFIELD_ < '123'");
+
+        // LE
+        expandTerms("_ANYFIELD_ <= '123'", "_ANYFIELD_ <= '123'");
+
+        // GT
+        expandTerms("_ANYFIELD_ > '123'", "_ANYFIELD_ > '123'");
+
+        // GE
+        expandTerms("_ANYFIELD_ >= '123'", "_ANYFIELD_ >= '123'");
+    }
+
     private void expandTerms(String original, String expected) throws ParseException {
         ASTJexlScript script = JexlASTHelper.parseJexlQuery(original);
         ASTJexlScript expanded = ExpandMultiNormalizedTerms.expandTerms(config, helper, script);
diff --git a/warehouse/query-core/src/test/resources/datawave/query/QueryLogicFactory.xml b/warehouse/query-core/src/test/resources/datawave/query/QueryLogicFactory.xml
@@ -299,6 +299,13 @@
                 <value>datawave.query.enrich.DatawaveTermFrequencyEnricher</value>
             </list>
         </property>
+
+        <property name="excludeUnfieldedTypes">
+            <list value-type="datawave.data.type.Type">
+                <bean class="datawave.data.type.NumberType" />
+            </list>
+        </property>
+
         <property name="useFilters" value="false" />
         <property name="filterClassNames" value=""/>
 

Original file line number	Diff line number	Diff line change
`@@ -2412,6 +2412,14 @@ public void setIvaratorCacheScanTimeoutMinutes(long hdfsCacheScanTimeoutMinutes)`
`2412`	`2412`	`getConfig().setIvaratorCacheScanTimeout(hdfsCacheScanTimeoutMinutes * 1000 * 60);`
`2413`	`2413`	`}`
`2414`	`2414`
	`2415`	`+ public List<Type<?>> getExcludeUnfieldedTypes() {`
	`2416`	`+ return getConfig().getExcludeUnfieldedTypes();`
	`2417`	`+ }`
	`2418`	`+`
	`2419`	`+ public void setExcludeUnfieldedTypes(List<Type<?>> excludeUnfieldedTypes) {`
	`2420`	`+ getConfig().setExcludeUnfieldedTypes(excludeUnfieldedTypes);`
	`2421`	`+ }`
	`2422`	`+`
`2415`	`2423`	`public String getHdfsSiteConfigURLs() {`
`2416`	`2424`	`return getConfig().getHdfsSiteConfigURLs();`
`2417`	`2425`	`}`