Module org.elasticsearch.xcore

Package org.elasticsearch.xpack.core.ml.inference.trainedmodel

Class Tokenization

java.lang.Object

org.elasticsearch.xpack.core.ml.inference.trainedmodel.Tokenization

All Implemented Interfaces:

NamedWriteable, Writeable, ToXContent, ToXContentObject, NamedXContentObject

Direct Known Subclasses:

BertJapaneseTokenization, BertTokenization, ByteLevelBpeTokenization, DebertaV2Tokenization, MPNetTokenization, RobertaTokenization, XLMRobertaTokenization

public abstract class Tokenization
extends Object
implements NamedXContentObject, NamedWriteable

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
static final record	Tokenization.SpanSettings
static enum	Tokenization.Truncate

Nested classes/interfaces inherited from interface org.elasticsearch.xcontent.ToXContent

ToXContent.DelegatingMapParams, ToXContent.MapParams, ToXContent.Params

Nested classes/interfaces inherited from interface org.elasticsearch.common.io.stream.Writeable

Writeable.Reader<V>, Writeable.Writer<V>

Field Summary

Fields

Modifier and Type	Field	Description
static final int	DEFAULT_MAX_SEQUENCE_LENGTH
static final ParseField	DO_LOWER_CASE
protected final boolean	doLowerCase
static final ParseField	MAX_SEQUENCE_LENGTH
protected final int	maxSequenceLength
protected final int	span
static final ParseField	SPAN
protected final Tokenization.Truncate	truncate
static final ParseField	TRUNCATE
static final int	UNSET_SPAN_VALUE
static final ParseField	WITH_SPECIAL_TOKENS
protected final boolean	withSpecialTokens

Fields inherited from interface org.elasticsearch.xcontent.ToXContent

EMPTY, EMPTY_PARAMS

Constructor Summary

Constructors

Constructor	Description
Tokenization(StreamInput in)

Method Summary

Modifier and Type	Method	Description
static BertTokenization	createDefault()
boolean	doLowerCase()
boolean	equals(Object o)
abstract String	getMaskToken()
int	getMaxSequenceLength()
int	getSpan()
Tokenization.Truncate	getTruncate()
int	hashCode()
int	maxSequenceLength()
XContentBuilder	toXContent(XContentBuilder builder, ToXContent.Params params)
Tokenization	updateWindowSettings(Tokenization.SpanSettings update)	Return a copy of this with the tokenizer span settings updated
static void	validateSpanAndMaxSequenceLength(int maxSequenceLength, int span)
static void	validateSpanAndTruncate(Tokenization.Truncate truncate, Integer span)
void	validateVocabulary(PutTrainedModelVocabularyAction.Request request)
boolean	withSpecialTokens()
void	writeTo(StreamOutput out)

Methods inherited from class java.lang.Object

clone, finalize, getClass, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.elasticsearch.common.io.stream.NamedWriteable

getWriteableName

Methods inherited from interface org.elasticsearch.xpack.core.ml.utils.NamedXContentObject

getName

Methods inherited from interface org.elasticsearch.xcontent.ToXContentObject

isFragment

Field Details

DO_LOWER_CASE

public static final ParseField DO_LOWER_CASE

WITH_SPECIAL_TOKENS

public static final ParseField WITH_SPECIAL_TOKENS

MAX_SEQUENCE_LENGTH

public static final ParseField MAX_SEQUENCE_LENGTH

TRUNCATE

public static final ParseField TRUNCATE

SPAN

public static final ParseField SPAN

DEFAULT_MAX_SEQUENCE_LENGTH

public static final int DEFAULT_MAX_SEQUENCE_LENGTH

See Also:

• Constant Field Values

UNSET_SPAN_VALUE

public static final int UNSET_SPAN_VALUE

See Also:

• Constant Field Values

doLowerCase

protected final boolean doLowerCase

withSpecialTokens

protected final boolean withSpecialTokens

maxSequenceLength

protected final int maxSequenceLength

truncate

protected final Tokenization.Truncate truncate

span

protected final int span

Constructor Details

Tokenization

public Tokenization(StreamInput in)
             throws IOException

Throws:

IOException

Method Details

createDefault

public static BertTokenization createDefault()

updateWindowSettings

public Tokenization updateWindowSettings(Tokenization.SpanSettings update)

Return a copy of this with the tokenizer span settings updated

Parameters:

update - The settings to update

Returns:

An updated Tokenization

writeTo

public void writeTo(StreamOutput out)
             throws IOException

Specified by:

writeTo in interface Writeable

Throws:

IOException

getMaskToken

public abstract String getMaskToken()

toXContent

public XContentBuilder toXContent(XContentBuilder builder,
 ToXContent.Params params)
                           throws IOException

Specified by:

toXContent in interface ToXContent

Throws:

IOException

validateSpanAndMaxSequenceLength

public static void validateSpanAndMaxSequenceLength(int maxSequenceLength,
 int span)

validateSpanAndTruncate

public static void validateSpanAndTruncate(@Nullable
 Tokenization.Truncate truncate,
 @Nullable
 Integer span)

equals

public boolean equals(Object o)

Overrides:

equals in class Object

hashCode

public int hashCode()

Overrides:

hashCode in class Object

doLowerCase

public boolean doLowerCase()

withSpecialTokens

public boolean withSpecialTokens()

maxSequenceLength

public int maxSequenceLength()

getTruncate

public Tokenization.Truncate getTruncate()

getSpan

public int getSpan()

getMaxSequenceLength

public int getMaxSequenceLength()

validateVocabulary

public void validateVocabulary(PutTrainedModelVocabularyAction.Request request)