java.lang.Object

org.elasticsearch.simdvec.ESVectorUtil

public class ESVectorUtil extends Object

Constructor Summary

Constructors

Constructor

Description

ESVectorUtil()
Method Summary

Modifier and Type

Method

Description

static int

andBitCount(byte[] a, byte[] b)

AND bit count computed over signed bytes.

static void

bFloat16ToFloat(byte[] bfBytes, int bfOffset, float[] floats, int floatOffset, int floatCount, ByteOrder byteOrder)

static void

calculateOSQGridPoints(float[] target, int[] quantize, int points, float[] pts)

Calculate the grid points for optimized-scalar quantization

static float

calculateOSQLoss(float[] target, float lowerInterval, float upperInterval, int points, float norm2, float lambda, int[] quantize)

Calculate the loss for optimized-scalar quantization for the given parameteres

static void

centerAndCalculateOSQStatsDp(byte[] target, byte[] centroid, float[] centered, float[] stats)

Center the byte target vector against a byte centroid and calculate the optimized-scalar quantization statistics for dot-product similarity.

static void

centerAndCalculateOSQStatsDp(float[] target, float[] centroid, float[] centered, float[] stats)

Center the target vector and calculate the optimized-scalar quantization statistics

static void

centerAndCalculateOSQStatsEuclidean(byte[] target, byte[] centroid, float[] centered, float[] stats)

Center the byte target vector against a byte centroid and calculate the optimized-scalar quantization statistics for euclidean similarity.

static void

centerAndCalculateOSQStatsEuclidean(float[] target, float[] centroid, float[] centered, float[] stats)

Center the target vector and calculate the optimized-scalar quantization statistics

static int

codePointCount(org.apache.lucene.util.BytesRef bytesRef)

Count the number of Unicode code points in a utf-8 encoded string.

static boolean

contains(byte[] value, int valueOffset, int valueLength, byte[] term, int termOffset, int termLength)

Checks whether the byte sequence term appears as a contiguous subsequence within value.

static float

cosine(byte[] a, byte[] b)

static void

cosineBulk(byte[] q, byte[] v0, byte[] v1, byte[] v2, byte[] v3, int distancesOffset, float[] distances)

Bulk computation of cosine similarity from a byte query vector to four byte candidate vectors.

static float

dotProduct(byte[] a, byte[] b)

static float

dotProduct(byte[] a, byte[] b, int length)

Dot product of the first length components of a and b.

static float

dotProduct(byte[] a, byte[] b, int offset, int length)

Dot product over [offset, offset + length).

static float

dotProduct(float[] a, float[] b)

static float

dotProduct(float[] a, float[] b, int length)

Dot product of the first length components of a and b.

static float

dotProduct(float[] a, float[] b, int offset, int length)

Dot product over [offset, offset + length).

static void

dotProductBulk(byte[] q, byte[] v0, byte[] v1, byte[] v2, byte[] v3, int distancesOffset, float[] distances)

Bulk computation of dot product from a byte query vector to four byte candidate vectors.

static void

dotProductBulk(float[] q, float[] v0, float[] v1, float[] v2, float[] v3, int distancesOffset, float[] distances)

Bulk computation of square distances between a query vector and four vectors.Result is stored in the provided distances array.

static void

floatToBFloat16(float[] floats, int floatOffset, byte[] bfBytes, int bfOffset, int floatCount, ByteOrder byteOrder)

static ES91OSQVectorsScorer

getES91OSQVectorsScorer(org.apache.lucene.store.IndexInput input, int dimension, int bulkSize)

static ES92Int7VectorsScorer

getES92Int7VectorsScorer(org.apache.lucene.store.IndexInput input, int dimension, int bulkSize)

static ES93BinaryQuantizedVectorScorer

getES93BinaryQuantizedVectorScorer(org.apache.lucene.store.IndexInput input, int dimension, int vectorLengthInBytes)

static ES940OSQVectorsScorer

getES940OSQVectorsScorer(org.apache.lucene.store.IndexInput input, byte queryBits, byte indexBits, int dimension, int dataLength, int bulkSize, ES940OSQVectorsScorer.BitEncoding bitEncoding)

static int

indexOf(byte[] bytes, int offset, int length, byte marker)

Searches for the first occurrence of the given marker byte in the specified range of the array.

static void

inRangeBitmask(long[] values, long lowerValue, long upperValue, long[] matches)

For every index i in [0, values.length), sets bit i in matches (matches[i>>>6], bit position i & 0x3f) when values[i] lies in [lowerValue, upperValue].

static long

ipByteBinByte(byte[] q, byte[] d)

static int

ipByteBit(byte[] q, byte[] d)

Compute the inner product of two vectors, where the query vector is a byte vector and the document vector is a bit vector.

static float

ipFloatBit(float[] q, byte[] d)

Compute the inner product of two vectors, where the query vector is a float vector and the document vector is a bit vector.

static float

ipFloatByte(float[] q, byte[] d)

Compute the inner product of two vectors, where the query vector is a float vector and the document vector is a byte vector.

static void

l2Normalize(byte[] v)

L2-normalizes all components of v in place.

static void

l2Normalize(byte[] v, int length)

L2-normalizes the prefix v[0..length) in place using signed byte values as real components.

static void

l2Normalize(byte[] v, int offset, int length)

L2-normalizes v[offset:offset + length) in place using signed byte values as real components.

static void

l2Normalize(float[] v)

L2-normalizes all components of v in place.

static void

l2Normalize(float[] v, int length)

L2-normalizes the prefix v[0..length) in place.

static void

l2Normalize(float[] v, int offset, int length)

L2-normalizes v[offset:offset + length) in place.

static void

linearCombination(float scaleOther, byte[] other, float[] dest)

Computes dest = scale * other + dest, widening byte src to float.

static void

linearCombination(float scaleOther, byte[] other, float scaleDest, float[] dest)

Computes dest[d] = scaleSrc * src[d] + scaleDest * dest[d], widening byte src to float.

static void

linearCombination(float scaleOther, float[] other, float[] dest)

Computes dest = scale * other + dest

static void

linearCombination(float scaleOther, float[] other, float scaleDest, float[] dest)

Computes dest = scale * other + scaledDes * dest

static float

logSumExpNQT(float[] vector)

Calculates an approximation of the LogSumExp of the input array in base 2.

static float

logSumExpNQTDiff(float[] v1, float[] v2, float eps)

Calculates a shifted and scaled LogSumExp of the input arrays in base 2, according to the formula: log2(sum_i(pow(2, (v1[i] - v2[i]) / eps))) This implementation uses the log-sum-exp trick for numerical stability and Not-Quite-Trascendental functions for speed.

static float

max(float[] values, int length)

static float

maxSimDotProduct(MultiBFloat16VectorsSource source, float[][] query, float[] scoresScratch)

Computes max-sim dot product for float query vectors against a bfloat16 multi-vector source.

static float

maxSimDotProduct(MultiByteVectorsSource source, byte[][] query, float[] scoresScratch)

Computes max-sim dot product for byte query vectors against a multi-vector source.

static float

maxSimDotProduct(MultiFloatVectorsSource source, float[][] query, float[] scoresScratch)

Computes max-sim dot product for float query vectors against a multi-vector source.

static void

packAsBinary(int[] vector, byte[] packed)

Packs the provided int array populated with "0" and "1" values into a byte array.

static void

packAsBytes(int[] src, byte[] dst, int len)

Narrows each of the first len ints to a byte by truncating to the low 8 bits, writing into dst[0..len).

static void

packDibit(int[] vector, byte[] packed)

static void

packDibitQuad(int[] vector, byte[] packed)

static void

pow2DiffAndScaleNQT(float[] v1, float[] v2, float a, float eps, float[] result)

Compute the following operation: result[i] = pow(2, (a + v1[i] - v2[i]) / eps) This implementation uses the log-sum-exp trick for numerical stability.

static int

quantizeVectorWithIntervals(float[] vector, int[] destination, float lowInterval, float upperInterval, byte bit)

Optimized-scalar quantization of the provided vector to the provided destination array.

static float

soarDistance(byte[] v1, byte[] centroid, float[] originalResidual, float soarLambda, float rnorm)

Compute the SOAR distance between a byte vector and a byte centroid.

static float

soarDistance(float[] v1, float[] centroid, float[] originalResidual, float soarLambda, float rnorm)

calculates the soar distance for a vector and a centroid

static void

soarDistanceBulk(byte[] v1, byte[] c0, byte[] c1, byte[] c2, byte[] c3, float[] originalResidual, float soarLambda, float rnorm, float[] distances)

Calculates SOAR distances between a byte query vector and 4 byte centroid vectors in bulk.

static void

soarDistanceBulk(float[] v1, float[] c0, float[] c1, float[] c2, float[] c3, float[] originalResidual, float soarLambda, float rnorm, float[] distances)

Bulk computation of the soar distance for a vector to four centroids

static float

squareDistance(byte[] a, byte[] b)

static float

squareDistance(byte[] a, byte[] b, int offset, int length)

Returns the sum of squared differences of the two byte vectors over a sub-range.

static float

squareDistance(float[] a, float[] b)

static float

squareDistance(float[] a, float[] b, int offset, int length)

static void

squareDistanceBulk(byte[] q, byte[] v0, byte[] v1, byte[] v2, byte[] v3, int distancesOffset, float[] distances)

Bulk computation of square distances from a byte query vector to four byte candidate vectors.

static void

squareDistanceBulk(byte[] q, int qOffset, int length, byte[] v0, byte[] v1, byte[] v2, byte[] v3, float[] distances)

Bulk computation of square distances from a sub-range of a byte query vector to four byte candidate vectors.

static void

squareDistanceBulk(float[] q, float[] v0, float[] v1, float[] v2, float[] v3, int distancesOffset, float[] distances)

Bulk computation of square distances between a query vector and four vectors.Result is stored in the provided distances array.

static void

squareDistanceBulk(float[] q, int qOffset, int length, float[] v0, float[] v1, float[] v2, float[] v3, float[] distances)

static void

subtract(float[] v1, float[] v2, float[] result)

Calculates the difference between two vectors and stores the result in a third vector.

static float

sum(float[] values, int length)

static void

transposeHalfByte(int[] q, byte[] quantQueryByte)

The idea here is to organize the query vector bits such that the first bit of every dimension is in the first set dimensions bits, or (dimensions/8) bytes.

Methods inherited from class java.lang.Object
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Details
- ESVectorUtil
  
  public ESVectorUtil()
Method Details
- getES91OSQVectorsScorer
  
  public static ES91OSQVectorsScorer getES91OSQVectorsScorer(org.apache.lucene.store.IndexInput input, int dimension, int bulkSize) throws IOException
  
  Throws:
  
  IOException
- getES940OSQVectorsScorer
  
  public static ES940OSQVectorsScorer getES940OSQVectorsScorer(org.apache.lucene.store.IndexInput input, byte queryBits, byte indexBits, int dimension, int dataLength, int bulkSize, ES940OSQVectorsScorer.BitEncoding bitEncoding) throws IOException
  
  Throws:
  
  IOException
- getES92Int7VectorsScorer
  
  public static ES92Int7VectorsScorer getES92Int7VectorsScorer(org.apache.lucene.store.IndexInput input, int dimension, int bulkSize) throws IOException
  
  Throws:
  
  IOException
- getES93BinaryQuantizedVectorScorer
  
  public static ES93BinaryQuantizedVectorScorer getES93BinaryQuantizedVectorScorer(org.apache.lucene.store.IndexInput input, int dimension, int vectorLengthInBytes) throws IOException
  
  Throws:
  
  IOException
- bFloat16ToFloat
  
  public static void bFloat16ToFloat(byte[] bfBytes, int bfOffset, float[] floats, int floatOffset, int floatCount, ByteOrder byteOrder)
- floatToBFloat16
  
  public static void floatToBFloat16(float[] floats, int floatOffset, byte[] bfBytes, int bfOffset, int floatCount, ByteOrder byteOrder)
- dotProduct
  
  public static float dotProduct(float[] a, float[] b)
- dotProduct
  
  public static float dotProduct(float[] a, float[] b, int length)
  
  Dot product of the first length components of a and b.
- dotProduct
  
  public static float dotProduct(float[] a, float[] b, int offset, int length)
  
  Dot product over [offset, offset + length).
- l2Normalize
  
  public static void l2Normalize(float[] v, int length)
  
  L2-normalizes the prefix v[0..length) in place. Elements at indices length and beyond are left unchanged. A zero prefix is a no-op; unlike VectorUtil.l2normalize(float[]), this method does not throw on a zero vector.
- l2Normalize
  
  public static void l2Normalize(float[] v, int offset, int length)
  
  L2-normalizes v[offset:offset + length) in place. Elements outside the range are left unchanged. A zero range is a no-op.
- l2Normalize
  
  public static void l2Normalize(float[] v)
  
  L2-normalizes all components of v in place.
- squareDistance
  
  public static float squareDistance(float[] a, float[] b)
- squareDistance
  
  public static float squareDistance(float[] a, float[] b, int offset, int length)
- cosine
  
  public static float cosine(byte[] a, byte[] b)
- dotProduct
  
  public static float dotProduct(byte[] a, byte[] b)
- dotProduct
  
  public static float dotProduct(byte[] a, byte[] b, int length)
  
  Dot product of the first length components of a and b.
- dotProduct
  
  public static float dotProduct(byte[] a, byte[] b, int offset, int length)
  
  Dot product over [offset, offset + length).
- l2Normalize
  
  public static void l2Normalize(byte[] v, int length)
  
  L2-normalizes the prefix v[0..length) in place using signed byte values as real components. Elements at indices length and beyond are left unchanged. A zero prefix is a no-op.
- l2Normalize
  
  public static void l2Normalize(byte[] v, int offset, int length)
  
  L2-normalizes v[offset:offset + length) in place using signed byte values as real components. Elements outside the range are left unchanged. A zero range is a no-op.
- l2Normalize
  
  public static void l2Normalize(byte[] v)
  
  L2-normalizes all components of v in place.
- maxSimDotProduct
  
  public static float maxSimDotProduct(MultiFloatVectorsSource source, float[][] query, float[] scoresScratch)
  
  Computes max-sim dot product for float query vectors against a multi-vector source.
  The provided scoresScratch buffer is reused as temporary per-document scores for each query vector to avoid per-call allocations. Its length must be at least source.vectorCount().
- maxSimDotProduct
  
  public static float maxSimDotProduct(MultiBFloat16VectorsSource source, float[][] query, float[] scoresScratch)
  
  Computes max-sim dot product for float query vectors against a bfloat16 multi-vector source.
  The provided scoresScratch buffer is reused as temporary per-document scores for each query vector to avoid per-call allocations. Its length must be at least source.vectorCount().
- maxSimDotProduct
  
  public static float maxSimDotProduct(MultiByteVectorsSource source, byte[][] query, float[] scoresScratch)
  
  Computes max-sim dot product for byte query vectors against a multi-vector source.
  The provided scoresScratch buffer is reused as temporary per-document scores for each query vector to avoid per-call allocations. Its length must be at least source.vectorCount().
- squareDistance
  
  public static float squareDistance(byte[] a, byte[] b)
- squareDistance
  
  public static float squareDistance(byte[] a, byte[] b, int offset, int length)
  
  Returns the sum of squared differences of the two byte vectors over a sub-range.
- squareDistanceBulk
  
  public static void squareDistanceBulk(byte[] q, byte[] v0, byte[] v1, byte[] v2, byte[] v3, int distancesOffset, float[] distances)
  
  Bulk computation of square distances from a byte query vector to four byte candidate vectors.
- squareDistanceBulk
  
  public static void squareDistanceBulk(byte[] q, int qOffset, int length, byte[] v0, byte[] v1, byte[] v2, byte[] v3, float[] distances)
  
  Bulk computation of square distances from a sub-range of a byte query vector to four byte candidate vectors.
- dotProductBulk
  
  public static void dotProductBulk(byte[] q, byte[] v0, byte[] v1, byte[] v2, byte[] v3, int distancesOffset, float[] distances)
  
  Bulk computation of dot product from a byte query vector to four byte candidate vectors.
- cosineBulk
  
  public static void cosineBulk(byte[] q, byte[] v0, byte[] v1, byte[] v2, byte[] v3, int distancesOffset, float[] distances)
  
  Bulk computation of cosine similarity from a byte query vector to four byte candidate vectors.
- ipByteBinByte
  
  public static long ipByteBinByte(byte[] q, byte[] d)
- ipByteBit
  
  public static int ipByteBit(byte[] q, byte[] d)
  
  Compute the inner product of two vectors, where the query vector is a byte vector and the document vector is a bit vector. This will return the sum of the query vector values using the document vector as a mask. When comparing the bits with the bytes, they are done in "big endian" order. For example, if the byte vector is [1, 2, 3, 4, 5, 6, 7, 8] and the bit vector is [0b10000000], the inner product will be 1.0.
  
  Parameters:
  
  q - the query vector
  
  d - the document vector
  
  Returns:
  
  the inner product of the two vectors
- ipFloatBit
  
  public static float ipFloatBit(float[] q, byte[] d)
  
  Compute the inner product of two vectors, where the query vector is a float vector and the document vector is a bit vector. This will return the sum of the query vector values using the document vector as a mask. When comparing the bits with the floats, they are done in "big endian" order. For example, if the float vector is [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0] and the bit vector is [0b10000000], the inner product will be 1.0.
  
  Parameters:
  
  q - the query vector
  
  d - the document vector
  
  Returns:
  
  the inner product of the two vectors
- ipFloatByte
  
  public static float ipFloatByte(float[] q, byte[] d)
  
  Compute the inner product of two vectors, where the query vector is a float vector and the document vector is a byte vector.
  
  Parameters:
  
  q - the query vector
  
  d - the document vector
  
  Returns:
  
  the inner product of the two vectors
- andBitCount
  
  public static int andBitCount(byte[] a, byte[] b)
  
  AND bit count computed over signed bytes. Copied from Lucene's XOR implementation
  
  Parameters:
  
  a - bytes containing a vector
  
  b - bytes containing another vector, of the same dimension
  
  Returns:
  
  the value of the AND bit count of the two vectors
- max
  
  public static float max(float[] values, int length)
- sum
  
  public static float sum(float[] values, int length)
- calculateOSQLoss
  
  public static float calculateOSQLoss(float[] target, float lowerInterval, float upperInterval, int points, float norm2, float lambda, int[] quantize)
  
  Calculate the loss for optimized-scalar quantization for the given parameteres
  
  Parameters:
  
  target - The vector being quantized, assumed to be centered
  
  lowerInterval - The lower interval value for which to calculate the loss
  
  upperInterval - The upper interval value for which to calculate the loss
  
  points - the quantization points
  
  norm2 - The norm squared of the target vector
  
  lambda - The lambda parameter for controlling anisotropic loss calculation
  
  quantize - array to store the computed quantize vector.
  
  Returns:
  
  The loss for the given parameters
- calculateOSQGridPoints
  
  public static void calculateOSQGridPoints(float[] target, int[] quantize, int points, float[] pts)
  
  Calculate the grid points for optimized-scalar quantization
  
  Parameters:
  
  target - The vector being quantized, assumed to be centered
  
  quantize - The quantize vector which should have at least the target vector length
  
  points - the quantization points
  
  pts - The array to store the grid points, must be of length 5
- centerAndCalculateOSQStatsEuclidean
  
  public static void centerAndCalculateOSQStatsEuclidean(float[] target, float[] centroid, float[] centered, float[] stats)
  
  Center the target vector and calculate the optimized-scalar quantization statistics
  
  Parameters:
  
  target - The vector being quantized
  
  centroid - The centroid of the target vector
  
  centered - The destination of the centered vector, will be overwritten
  
  stats - The array to store the statistics, must be of length 5
- centerAndCalculateOSQStatsDp
  
  public static void centerAndCalculateOSQStatsDp(float[] target, float[] centroid, float[] centered, float[] stats)
  
  Center the target vector and calculate the optimized-scalar quantization statistics
  
  Parameters:
  
  target - The vector being quantized
  
  centroid - The centroid of the target vector
  
  centered - The destination of the centered vector, will be overwritten
  
  stats - The array to store the statistics, must be of length 6
- centerAndCalculateOSQStatsEuclidean
  
  public static void centerAndCalculateOSQStatsEuclidean(byte[] target, byte[] centroid, float[] centered, float[] stats)
  
  Center the byte target vector against a byte centroid and calculate the optimized-scalar quantization statistics for euclidean similarity.
  
  Parameters:
  
  target - The byte vector being quantized
  
  centroid - The byte centroid of the target vector
  
  centered - The destination of the centered vector, will be overwritten
  
  stats - The array to store the statistics, must be of length 5
- centerAndCalculateOSQStatsDp
  
  public static void centerAndCalculateOSQStatsDp(byte[] target, byte[] centroid, float[] centered, float[] stats)
  
  Center the byte target vector against a byte centroid and calculate the optimized-scalar quantization statistics for dot-product similarity.
  
  Parameters:
  
  target - The byte vector being quantized
  
  centroid - The byte centroid of the target vector
  
  centered - The destination of the centered vector, will be overwritten
  
  stats - The array to store the statistics, must be of length 6
- subtract
  
  public static void subtract(float[] v1, float[] v2, float[] result)
  
  Calculates the difference between two vectors and stores the result in a third vector.
  
  Parameters:
  
  v1 - the first vector
  
  v2 - the second vector
  
  result - the result vector, must be the same length as the input vectors
- soarDistance
  
  public static float soarDistance(float[] v1, float[] centroid, float[] originalResidual, float soarLambda, float rnorm)
  
  calculates the soar distance for a vector and a centroid
  
  Parameters:
  
  v1 - the vector
  
  centroid - the centroid
  
  originalResidual - the residual with the actually nearest centroid
  
  soarLambda - the lambda parameter
  
  rnorm - distance to the nearest centroid
  
  Returns:
  
  the soar distance
- soarDistance
  
  public static float soarDistance(byte[] v1, byte[] centroid, float[] originalResidual, float soarLambda, float rnorm)
  
  Compute the SOAR distance between a byte vector and a byte centroid. SOAR distance: ||x-c||^2 + lambda * ((x-c1)^T (x-c))^2 / ||x-c1||^2
  
  Parameters:
  
  v1 - the byte vector
  
  centroid - the byte centroid
  
  originalResidual - the precomputed residual (x - c1) as float
  
  soarLambda - the SOAR lambda parameter
  
  rnorm - the squared norm of the original residual
  
  Returns:
  
  the SOAR distance
- quantizeVectorWithIntervals
  
  public static int quantizeVectorWithIntervals(float[] vector, int[] destination, float lowInterval, float upperInterval, byte bit)
  
  Optimized-scalar quantization of the provided vector to the provided destination array.
  
  Parameters:
  
  vector - the vector to quantize
  
  destination - the array to store the result
  
  lowInterval - the minimum value, lower values in the original array will be replaced by this value
  
  upperInterval - the maximum value, bigger values in the original array will be replaced by this value
  
  bit - the number of bits to use for quantization, must be between 1 and 8
  
  Returns:
  
  return the sum of all the elements of the resulting quantized vector.
- squareDistanceBulk
  
  public static void squareDistanceBulk(float[] q, float[] v0, float[] v1, float[] v2, float[] v3, int distancesOffset, float[] distances)
  
  Bulk computation of square distances between a query vector and four vectors.Result is stored in the provided distances array.
  
  Parameters:
  
  q - the query vector
  
  v0 - the first vector
  
  v1 - the second vector
  
  v2 - the third vector
  
  v3 - the fourth vector
  
  distancesOffset - offset to the location in the distances array where we want to store the 4 results, we require distancesOffset to be between 0 and distances.length - 4
  
  distances - an array to store the computed square distances, must have length >= 4
  
  Throws:
  
  IllegalArgumentException - if the dimensions of the vectors do not match or if the distances array does not have length 4
- dotProductBulk
  
  public static void dotProductBulk(float[] q, float[] v0, float[] v1, float[] v2, float[] v3, int distancesOffset, float[] distances)
  
  Bulk computation of square distances between a query vector and four vectors.Result is stored in the provided distances array.
  
  Parameters:
  
  q - the query vector
  
  v0 - the first vector
  
  v1 - the second vector
  
  v2 - the third vector
  
  v3 - the fourth vector
  
  distancesOffset - offset to the location in the distances array where we want to store the 4 results, we require distancesOffset to be between 0 and distances.length - 4
  
  distances - an array to store the computed square distances, must have length >= 4
  
  Throws:
  
  IllegalArgumentException - if the dimensions of the vectors do not match or if the distances array does not have length 4
- squareDistanceBulk
  
  public static void squareDistanceBulk(float[] q, int qOffset, int length, float[] v0, float[] v1, float[] v2, float[] v3, float[] distances)
- soarDistanceBulk
  
  public static void soarDistanceBulk(float[] v1, float[] c0, float[] c1, float[] c2, float[] c3, float[] originalResidual, float soarLambda, float rnorm, float[] distances)
  
  Bulk computation of the soar distance for a vector to four centroids
  
  Parameters:
  
  v1 - the vector
  
  c0 - the first centroid
  
  c1 - the second centroid
  
  c2 - the third centroid
  
  c3 - the fourth centroid
  
  originalResidual - the residual with the actually nearest centroid
  
  soarLambda - the lambda parameter
  
  rnorm - distance to the nearest centroid
  
  distances - an array to store the computed soar distances, must have length 4
- soarDistanceBulk
  
  public static void soarDistanceBulk(byte[] v1, byte[] c0, byte[] c1, byte[] c2, byte[] c3, float[] originalResidual, float soarLambda, float rnorm, float[] distances)
  
  Calculates SOAR distances between a byte query vector and 4 byte centroid vectors in bulk. SOAR distance: ||x-c||^2 + lambda * ((x-c)^T * originalResidual)^2 / rnorm
  
  Parameters:
  
  v1 - the query vector
  
  c0 - centroid 0
  
  c1 - centroid 1
  
  c2 - centroid 2
  
  c3 - centroid 3
  
  originalResidual - the precomputed residual (x - globalCentroid) as float
  
  soarLambda - the SOAR lambda parameter
  
  rnorm - the squared norm of the original residual
  
  distances - output array of length 4 for the computed distances
- packAsBytes
  
  public static void packAsBytes(int[] src, byte[] dst, int len)
  
  Narrows each of the first len ints to a byte by truncating to the low 8 bits, writing into dst[0..len). No bounds check is performed; the caller must ensure dst.length >= len.
  
  Parameters:
  
  src - int array of quantized values
  
  dst - byte array to receive the narrowed values
  
  len - number of elements to convert
- packAsBinary
  
  public static void packAsBinary(int[] vector, byte[] packed)
  
  Packs the provided int array populated with "0" and "1" values into a byte array.
  
  Parameters:
  
  vector - the int array to pack, must contain only "0" and "1" values.
  
  packed - the byte array to store the packed result, must be large enough to hold the packed data.
- packDibit
  
  public static void packDibit(int[] vector, byte[] packed)
- packDibitQuad
  
  public static void packDibitQuad(int[] vector, byte[] packed)
- transposeHalfByte
  
  public static void transposeHalfByte(int[] q, byte[] quantQueryByte)
  
  The idea here is to organize the query vector bits such that the first bit of every dimension is in the first set dimensions bits, or (dimensions/8) bytes. The second, third, and fourth bits are in the second, third, and fourth set of dimensions bits, respectively. This allows for direct bitwise comparisons with the stored index vectors through summing the bitwise results with the relative required bit shifts.
  
  Parameters:
  
  q - the query vector, assumed to be half-byte quantized with values between 0 and 15
  
  quantQueryByte - the byte array to store the transposed query vector.
- indexOf
  
  public static int indexOf(byte[] bytes, int offset, int length, byte marker)
  
  Searches for the first occurrence of the given marker byte in the specified range of the array.
  The search starts at offset and examines at most length bytes. The return value is the relative index of the first occurrence of marker within this slice, or -1 if not found.
  
  Parameters:
  
  bytes - the byte array to search
  
  offset - the starting index within the array
  
  length - the number of bytes to examine
  
  marker - the byte to search for
  
  Returns:
  
  the relative index (0..length-1) of the first match, or -1 if not found
- contains
  
  public static boolean contains(byte[] value, int valueOffset, int valueLength, byte[] term, int termOffset, int termLength)
  
  Checks whether the byte sequence term appears as a contiguous subsequence within value.
  
  Parameters:
  
  value - the byte array to search in
  
  valueOffset - the starting index within value
  
  valueLength - the number of bytes to search
  
  term - the byte array containing the term to search for
  
  termOffset - the starting index within term
  
  termLength - the number of bytes in the term
  
  Returns:
  
  true if term is found within value
- codePointCount
  
  public static int codePointCount(org.apache.lucene.util.BytesRef bytesRef)
  
  Count the number of Unicode code points in a utf-8 encoded string. Assumes that the input string is correctly encoded. If the input string is incorrectly encoded, no errors will be thrown, but invalid results will be returned.
  
  Parameters:
  
  bytesRef - bytes reference containing a valid utf-8 encoded string
  
  Returns:
  
  the number of code points in the bytes ref
- linearCombination
  
  public static void linearCombination(float scaleOther, float[] other, float scaleDest, float[] dest)
  
  Computes dest = scale * other + scaledDes * dest
  
  Parameters:
  
  scaleOther - a multiplicative factor for other
  
  other - the other vector
  
  scaleDest - a multiplicative factor for dest
  
  dest - the destination vector
- linearCombination
  
  public static void linearCombination(float scaleOther, float[] other, float[] dest)
  
  Computes dest = scale * other + dest
  
  Parameters:
  
  scaleOther - a multiplicative factor for other
  
  other - the other vector
  
  dest - the destination vector
- linearCombination
  
  public static void linearCombination(float scaleOther, byte[] other, float scaleDest, float[] dest)
  
  Computes dest[d] = scaleSrc * src[d] + scaleDest * dest[d], widening byte src to float.
  
  Parameters:
  
  scaleOther - a multiplicative factor for src
  
  other - the byte source vector (widened to float for computation)
  
  scaleDest - a multiplicative factor for dest
  
  dest - the destination float vector (modified in place)
- linearCombination
  
  public static void linearCombination(float scaleOther, byte[] other, float[] dest)
  
  Computes dest = scale * other + dest, widening byte src to float.
  
  Parameters:
  
  scaleOther - a multiplicative factor for src
  
  other - the byte source vector (widened to float for computation)
  
  dest - the destination float vector (modified in place)
- logSumExpNQT
  
  public static float logSumExpNQT(float[] vector)
  
  Calculates an approximation of the LogSumExp of the input array in base 2. The formula used is: log2(sum_i(pow(2, x[i]))). This implementation uses the log-sum-exp trick for numerical stability and Not-Quite-Trascendental functions for speed.
  
  Parameters:
  
  vector - The input array of double values (log probabilities/values).
  
  Returns:
  
  The log-sum-exp result.
- logSumExpNQTDiff
  
  public static float logSumExpNQTDiff(float[] v1, float[] v2, float eps)
  
  Calculates a shifted and scaled LogSumExp of the input arrays in base 2, according to the formula: log2(sum_i(pow(2, (v1[i] - v2[i]) / eps))) This implementation uses the log-sum-exp trick for numerical stability and Not-Quite-Trascendental functions for speed.
  
  Parameters:
  
  v1 - The first input array of double values (log probabilities/values).
  
  v2 - The second input array of double values (log probabilities/values).
  
  eps - The normalization constant (that is, the temperature parameter).
  
  Returns:
  
  The log-sum-exp result.
- pow2DiffAndScaleNQT
  
  public static void pow2DiffAndScaleNQT(float[] v1, float[] v2, float a, float eps, float[] result)
  
  Compute the following operation: result[i] = pow(2, (a + v1[i] - v2[i]) / eps) This implementation uses the log-sum-exp trick for numerical stability.
  
  Parameters:
  
  v1 - The first input array of double values (log probabilities/values).
  
  v2 - The second input array of double values (log probabilities/values).
  
  eps - The normalization constant (that is, the temperature parameter).
  
  result - The output array.
- inRangeBitmask
  
  public static void inRangeBitmask(long[] values, long lowerValue, long upperValue, long[] matches)
  
  For every index i in [0, values.length), sets bit i in matches (matches[i>>>6], bit position i & 0x3f) when values[i] lies in [lowerValue, upperValue].
  Requires values.length to be a multiple of 8 (the maximum supported SIMD lane count, for AVX-512) and matches.length == values.length / 64.

Class ESVectorUtil

Constructor Summary

Method Summary

Methods inherited from class java.lang.Object

Constructor Details

ESVectorUtil

Method Details

getES91OSQVectorsScorer

getES940OSQVectorsScorer

getES92Int7VectorsScorer

getES93BinaryQuantizedVectorScorer

bFloat16ToFloat

floatToBFloat16

dotProduct

dotProduct

dotProduct

l2Normalize

l2Normalize

l2Normalize

squareDistance

squareDistance

cosine

dotProduct

dotProduct

dotProduct

l2Normalize

l2Normalize

l2Normalize

maxSimDotProduct

maxSimDotProduct

maxSimDotProduct

squareDistance

squareDistance

squareDistanceBulk

squareDistanceBulk

dotProductBulk

cosineBulk

ipByteBinByte

ipByteBit

ipFloatBit

ipFloatByte

andBitCount

max

sum

calculateOSQLoss

calculateOSQGridPoints

centerAndCalculateOSQStatsEuclidean

centerAndCalculateOSQStatsDp

centerAndCalculateOSQStatsEuclidean

centerAndCalculateOSQStatsDp

subtract

soarDistance

soarDistance

quantizeVectorWithIntervals

squareDistanceBulk

dotProductBulk

squareDistanceBulk

soarDistanceBulk

soarDistanceBulk

packAsBytes

packAsBinary

packDibit

packDibitQuad

transposeHalfByte

indexOf

contains

codePointCount

linearCombination

linearCombination

linearCombination

linearCombination

logSumExpNQT

logSumExpNQTDiff

pow2DiffAndScaleNQT

inRangeBitmask