Histogram field block loader (#139308)

Adds a block loader for the histogram field type. This is the first step towards getting ESQL support for the type, although this PR doesn't add any user-facing functionality. As is tradition, most of the work here is writing the data generation for the randomized field testing framework.
---------

Co-authored-by: elasticsearchmachine <infra-root+elasticsearchmachine@elastic.co>

Author: not-napoleon

server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/BytesRefsFromBinaryBlockLoader.java

@@ -17,6 +17,11 @@
 
 import java.io.IOException;
 
+/**
+ * This block loader should be used for fields that are directly encoded as binary values but are always single valued, such as the
+ * histogram fields.  See also {@link BytesRefsFromCustomBinaryBlockLoader} for multivalued binary fields, and
+ * {@link BytesRefsFromOrdsBlockLoader} for ordinals-based binary values
+ */
 public class BytesRefsFromBinaryBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader {
 
     private final String fieldName;

server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/BytesRefsFromCustomBinaryBlockLoader.java

@@ -18,6 +18,11 @@
 
 import java.io.IOException;
 
+/**
+ * This block loader should be used for "wildcard-style" binary values, which is to say fields we have encoded into a binary
+ * format that supports multivalued via an encoding on our side.  See also {@link BytesRefsFromOrdsBlockLoader} for ordinals
+ * based multivalue aware binary fields, and {@link BytesRefsFromBinaryBlockLoader} for single-valued binary fields.
+ */
 public class BytesRefsFromCustomBinaryBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader {
     private final String fieldName;
 

server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/BytesRefsFromOrdsBlockLoader.java

@@ -17,7 +17,9 @@
 import java.io.IOException;
 
 /**
- * Loads {@code keyword} style fields that are stored as a lookup table and ordinals.
+ * Loads {@code keyword} style fields that are stored as a lookup table and ordinals.  See also {@link BytesRefsFromCustomBinaryBlockLoader}
+ * for {@code wildcard} style (i.e. non-ordinal encoded multivalued) and {@link BytesRefsFromBinaryBlockLoader} for {@code histogram}
+ * style (i.e. non-ordinal single valued).
  */
 public class BytesRefsFromOrdsBlockLoader extends AbstractBytesRefsFromOrdsBlockLoader {
     public BytesRefsFromOrdsBlockLoader(String fieldName) {

test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java

@@ -229,6 +229,15 @@ public DataSourceResponse.ObjectMappingParametersGenerator handle(
             .build();
     }
 
+    /**
+     * For a given mapping and input value, compute the value that will be in the block.  Values are generated from the
+     * {@link DocumentGenerator}, and the behavior can be controled by writing a custom {@link DataSourceHandler}.
+     *
+     * @param fieldMapping Generated parameters for this field mapping
+     * @param value Generated input value to convert
+     * @param testContext Context information for the current test run
+     * @return The value that will be added to the block
+     */
     protected abstract Object expected(Map<String, Object> fieldMapping, Object value, TestContext testContext);
 
     protected static Object maybeFoldList(List<?> list) {

x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java

@@ -29,6 +29,7 @@
 import org.elasticsearch.index.fielddata.IndexHistogramFieldData;
 import org.elasticsearch.index.fielddata.LeafHistogramFieldData;
 import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
+import org.elasticsearch.index.mapper.BlockLoader;
 import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader;
 import org.elasticsearch.index.mapper.DocumentParserContext;
 import org.elasticsearch.index.mapper.DocumentParsingException;
@@ -42,6 +43,7 @@
 import org.elasticsearch.index.mapper.SourceValueFetcher;
 import org.elasticsearch.index.mapper.TimeSeriesParams;
 import org.elasticsearch.index.mapper.ValueFetcher;
+import org.elasticsearch.index.mapper.blockloader.docvalues.BytesRefsFromBinaryBlockLoader;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.script.field.DocValuesScriptFieldFactory;
 import org.elasticsearch.search.DocValueFormat;
@@ -56,6 +58,7 @@
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.util.List;
 import java.util.Map;
 
 import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
@@ -305,6 +308,11 @@ public Query termQuery(Object value, SearchExecutionContext context) {
                 "[" + CONTENT_TYPE + "] field do not support searching, " + "use dedicated aggregations instead: [" + name() + "]"
             );
         }
+
+        @Override
+        public BlockLoader blockLoader(BlockLoaderContext blContext) {
+            return new BytesRefsFromBinaryBlockLoader(name());
+        }
     }
 
     @Override
@@ -351,17 +359,11 @@ public void parse(DocumentParserContext context) throws IOException {
                 parsedHistogram = HistogramParser.parse(fullPath(), subParser);
             }
 
-            BytesStreamOutput streamOutput = new BytesStreamOutput();
-            for (int i = 0; i < parsedHistogram.values().size(); i++) {
-                long count = parsedHistogram.counts().get(i);
-                assert count >= 0;
-                // we do not add elements with count == 0
-                if (count > 0) {
-                    streamOutput.writeVLong(count);
-                    streamOutput.writeLong(Double.doubleToRawLongBits(parsedHistogram.values().get(i)));
-                }
-            }
-            BytesRef docValue = streamOutput.bytes().toBytesRef();
+            List<Double> values = parsedHistogram.values();
+            List<Long> counts = parsedHistogram.counts();
+
+            BytesRef docValue = encodeBytesRef(values, counts);
+
             Field field = new BinaryDocValuesField(fullPath(), docValue);
             if (context.doc().getByKey(fieldType().name()) != null) {
                 throw new IllegalArgumentException(
@@ -402,6 +404,22 @@ public void parse(DocumentParserContext context) throws IOException {
         context.path().remove();
     }
 
+    static BytesRef encodeBytesRef(List<Double> values, List<Long> counts) throws IOException {
+        BytesStreamOutput streamOutput = new BytesStreamOutput();
+        assert counts.size() == values.size();
+        for (int i = 0; i < values.size(); i++) {
+            long count = counts.get(i);
+            assert count >= 0;
+            // we do not add elements with count == 0
+            if (count > 0) {
+                streamOutput.writeVLong(count);
+                streamOutput.writeLong(Double.doubleToRawLongBits(values.get(i)));
+            }
+        }
+        BytesRef docValue = streamOutput.bytes().toBytesRef();
+        return docValue;
+    }
+
     /** re-usable {@link HistogramValue} implementation */
     static class InternalHistogramValue extends HistogramValue {
         double value;

x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldBlockLoaderTests.java

@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.analytics.mapper;
+
+import org.elasticsearch.core.Types;
+import org.elasticsearch.datageneration.datasource.DataSourceHandler;
+import org.elasticsearch.datageneration.datasource.DataSourceRequest;
+import org.elasticsearch.datageneration.datasource.DataSourceResponse;
+import org.elasticsearch.index.mapper.BlockLoaderTestCase;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.analytics.AnalyticsPlugin;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+public class HistogramFieldBlockLoaderTests extends BlockLoaderTestCase {
+
+    public HistogramFieldBlockLoaderTests(Params params) {
+        super(HistogramFieldMapper.CONTENT_TYPE, List.of(DATA_SOURCE_HANDLER), params);
+    }
+
+    @Override
+    protected Collection<? extends Plugin> getPlugins() {
+        return List.of(new AnalyticsPlugin());
+    }
+
+    private static DataSourceHandler DATA_SOURCE_HANDLER = new DataSourceHandler() {
+
+        @Override
+        public DataSourceResponse.ObjectArrayGenerator handle(DataSourceRequest.ObjectArrayGenerator request) {
+            // histogram does not support multiple values in a document so we can't have object arrays
+            return new DataSourceResponse.ObjectArrayGenerator(Optional::empty);
+        }
+
+        @Override
+        public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) {
+            if (request.fieldType().equals(HistogramFieldMapper.CONTENT_TYPE) == false) {
+                return null;
+            }
+
+            return new DataSourceResponse.LeafMappingParametersGenerator(() -> {
+                var map = new HashMap<String, Object>();
+                if (ESTestCase.randomBoolean()) {
+                    map.put("ignore_malformed", ESTestCase.randomBoolean());
+                }
+                return map;
+            });
+        }
+
+        @Override
+        public DataSourceResponse.FieldDataGenerator handle(DataSourceRequest.FieldDataGenerator request) {
+            if (request.fieldType().equals(HistogramFieldMapper.CONTENT_TYPE) == false) {
+                return null;
+            }
+            return new DataSourceResponse.FieldDataGenerator(mapping -> {
+                List<Double> values = randomList(randomIntBetween(1, 1000), ESTestCase::randomDouble);
+                values.sort(Double::compareTo);
+                return Map.of(
+                    "values",
+                    values,
+                    "counts",
+                    // Note - we need the three parameter version of random list here to ensure it's always the same length as values
+                    randomList(values.size(), values.size(), ESTestCase::randomNonNegativeLong)
+                );
+            });
+        }
+    };
+
+    @Override
+    public void testBlockLoaderOfMultiField() throws IOException {
+        // Multi fields are not supported
+    }
+
+    @Override
+    protected Object expected(Map<String, Object> fieldMapping, Object value, TestContext testContext) {
+        Map<String, Object> valueAsMap = Types.forciblyCast(value);
+        List<Double> bucketValues = Types.forciblyCast(valueAsMap.get("values"));
+        List<Long> counts = Types.forciblyCast(valueAsMap.get("counts"));
+
+        try {
+            return HistogramFieldMapper.encodeBytesRef(bucketValues, counts);
+        } catch (IOException e) {
+            fail("failed to encode histogram field values");
+        }
+        throw new IllegalStateException("Unreachable");
+    }
+}