com.theokanning.openai.utils

Class TikTokensUtil

java.lang.Object

com.theokanning.openai.utils.TikTokensUtil

public class TikTokensUtil
extends java.lang.Object

Token calculation tool class

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	TikTokensUtil.ModelEnum

Constructor Summary

Constructors

Constructor and Description
TikTokensUtil()

Method Summary

Modifier and Type	Method and Description
static java.lang.String	decode(com.knuddels.jtokkit.api.Encoding enc, java.util.List<java.lang.Integer> encoded) Reverse calculate text information through Encoding and encoded array
static java.lang.String	decode(com.knuddels.jtokkit.api.EncodingType encodingType, java.util.List<java.lang.Integer> encoded) Reverse the encoded array to get the string text using EncodingType and the encoded array.
static java.lang.String	decode(java.lang.String modelName, java.util.List<java.lang.Integer> encoded) Reverse the string text through the model name and the encoded array.
static java.util.List<java.lang.Integer>	encode(com.knuddels.jtokkit.api.Encoding enc, java.lang.String text) Get encoding array through Encoding and text.
static java.util.List<java.lang.Integer>	encode(com.knuddels.jtokkit.api.EncodingType encodingType, java.lang.String text) Obtain the encoding array by encoding;
static java.util.List<java.lang.Integer>	encode(java.lang.String modelName, java.lang.String text) Get the encoded array by model name using encode.
static com.knuddels.jtokkit.api.Encoding	getEncoding(com.knuddels.jtokkit.api.EncodingType encodingType) Get an Encoding object by Encoding type
static com.knuddels.jtokkit.api.Encoding	getEncoding(java.lang.String modelName) Get an Encoding object by model name.
static com.knuddels.jtokkit.api.ModelType	getModelTypeByName(java.lang.String name) Obtain the modelType.
static boolean	isBlank(java.lang.CharSequence str)
static boolean	isBlankChar(char c)
static boolean	isBlankChar(int c)
static boolean	isNotBlank(java.lang.CharSequence str)
static int	tokens(com.knuddels.jtokkit.api.Encoding enc, java.lang.String text) Calculate tokens of text information through Encoding.
static int	tokens(com.knuddels.jtokkit.api.EncodingType encodingType, java.lang.String text) Compute the tokens of the specified string through EncodingType.
static int	tokens(java.lang.String modelName, java.util.List<ChatMessage> messages) Calculate the encoded array for messages by model name.
static int	tokens(java.lang.String modelName, java.lang.String text) Calculate the tokens of a specified string by model name.

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

TikTokensUtil

public TikTokensUtil()

Method Detail

encode

public static java.util.List<java.lang.Integer> encode(com.knuddels.jtokkit.api.Encoding enc,
                                                       java.lang.String text)

Get encoding array through Encoding and text.

Parameters:

enc - Encoding type

text - Text information

Returns:

Encoding array

tokens

public static int tokens(com.knuddels.jtokkit.api.Encoding enc,
                         java.lang.String text)

Calculate tokens of text information through Encoding.

Parameters:

enc - Encoding type

text - Text information

Returns:

Number of tokens

decode

public static java.lang.String decode(com.knuddels.jtokkit.api.Encoding enc,
                                      java.util.List<java.lang.Integer> encoded)

Reverse calculate text information through Encoding and encoded array

Parameters:

enc - Encoding

encoded - Encoding array

Returns:

Text information corresponding to the encoding array.

getEncoding

public static com.knuddels.jtokkit.api.Encoding getEncoding(com.knuddels.jtokkit.api.EncodingType encodingType)

Get an Encoding object by Encoding type

Parameters:

encodingType -

Returns:

Encoding

encode

public static java.util.List<java.lang.Integer> encode(com.knuddels.jtokkit.api.EncodingType encodingType,
                                                       java.lang.String text)

Obtain the encoding array by encoding;

Parameters:

text -

Returns:

Encoding array

tokens

public static int tokens(com.knuddels.jtokkit.api.EncodingType encodingType,
                         java.lang.String text)

Compute the tokens of the specified string through EncodingType.

Parameters:

encodingType -

text -

Returns:

Number of tokens

decode

public static java.lang.String decode(com.knuddels.jtokkit.api.EncodingType encodingType,
                                      java.util.List<java.lang.Integer> encoded)

Reverse the encoded array to get the string text using EncodingType and the encoded array.

Parameters:

encodingType -

encoded -

Returns:

The string corresponding to the encoding array.

getEncoding

public static com.knuddels.jtokkit.api.Encoding getEncoding(java.lang.String modelName)

Get an Encoding object by model name.

Parameters:

modelName -

Returns:

Encoding

encode

public static java.util.List<java.lang.Integer> encode(java.lang.String modelName,
                                                       java.lang.String text)

Get the encoded array by model name using encode.

Parameters:

text - Text information

Returns:

Encoding array

tokens

public static int tokens(java.lang.String modelName,
                         java.lang.String text)

Calculate the tokens of a specified string by model name.

Parameters:

modelName -

text -

Returns:

Number of tokens

tokens

public static int tokens(java.lang.String modelName,
                         java.util.List<ChatMessage> messages)

Calculate the encoded array for messages by model name. Refer to the official processing logic: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb

Parameters:

modelName -

messages -

Returns:

Number of tokens

decode

public static java.lang.String decode(java.lang.String modelName,
                                      java.util.List<java.lang.Integer> encoded)

Reverse the string text through the model name and the encoded array.

Parameters:

modelName -

encoded -

Returns:

getModelTypeByName

public static com.knuddels.jtokkit.api.ModelType getModelTypeByName(java.lang.String name)

Obtain the modelType.

Parameters:

name -

Returns:

isBlankChar

public static boolean isBlankChar(int c)

isBlankChar

public static boolean isBlankChar(char c)

isNotBlank

public static boolean isNotBlank(java.lang.CharSequence str)

isBlank

public static boolean isBlank(java.lang.CharSequence str)