public class TikTokensUtil
extends java.lang.Object
| Modifier and Type | Class and Description |
|---|---|
static class |
TikTokensUtil.ModelEnum |
| Constructor and Description |
|---|
TikTokensUtil() |
| Modifier and Type | Method and Description |
|---|---|
static java.lang.String |
decode(com.knuddels.jtokkit.api.Encoding enc,
java.util.List<java.lang.Integer> encoded)
Reverse calculate text information through Encoding and encoded array
|
static java.lang.String |
decode(com.knuddels.jtokkit.api.EncodingType encodingType,
java.util.List<java.lang.Integer> encoded)
Reverse the encoded array to get the string text using EncodingType and the encoded array.
|
static java.lang.String |
decode(java.lang.String modelName,
java.util.List<java.lang.Integer> encoded)
Reverse the string text through the model name and the encoded array.
|
static java.util.List<java.lang.Integer> |
encode(com.knuddels.jtokkit.api.Encoding enc,
java.lang.String text)
Get encoding array through Encoding and text.
|
static java.util.List<java.lang.Integer> |
encode(com.knuddels.jtokkit.api.EncodingType encodingType,
java.lang.String text)
Obtain the encoding array by encoding;
|
static java.util.List<java.lang.Integer> |
encode(java.lang.String modelName,
java.lang.String text)
Get the encoded array by model name using encode.
|
static com.knuddels.jtokkit.api.Encoding |
getEncoding(com.knuddels.jtokkit.api.EncodingType encodingType)
Get an Encoding object by Encoding type
|
static com.knuddels.jtokkit.api.Encoding |
getEncoding(java.lang.String modelName)
Get an Encoding object by model name.
|
static com.knuddels.jtokkit.api.ModelType |
getModelTypeByName(java.lang.String name)
Obtain the modelType.
|
static boolean |
isBlank(java.lang.CharSequence str) |
static boolean |
isBlankChar(char c) |
static boolean |
isBlankChar(int c) |
static boolean |
isNotBlank(java.lang.CharSequence str) |
static int |
tokens(com.knuddels.jtokkit.api.Encoding enc,
java.lang.String text)
Calculate tokens of text information through Encoding.
|
static int |
tokens(com.knuddels.jtokkit.api.EncodingType encodingType,
java.lang.String text)
Compute the tokens of the specified string through EncodingType.
|
static int |
tokens(java.lang.String modelName,
java.util.List<ChatMessage> messages)
Calculate the encoded array for messages by model name.
|
static int |
tokens(java.lang.String modelName,
java.lang.String text)
Calculate the tokens of a specified string by model name.
|
public static java.util.List<java.lang.Integer> encode(com.knuddels.jtokkit.api.Encoding enc,
java.lang.String text)
enc - Encoding typetext - Text informationpublic static int tokens(com.knuddels.jtokkit.api.Encoding enc,
java.lang.String text)
enc - Encoding typetext - Text informationpublic static java.lang.String decode(com.knuddels.jtokkit.api.Encoding enc,
java.util.List<java.lang.Integer> encoded)
enc - Encodingencoded - Encoding arraypublic static com.knuddels.jtokkit.api.Encoding getEncoding(com.knuddels.jtokkit.api.EncodingType encodingType)
encodingType - public static java.util.List<java.lang.Integer> encode(com.knuddels.jtokkit.api.EncodingType encodingType,
java.lang.String text)
text - public static int tokens(com.knuddels.jtokkit.api.EncodingType encodingType,
java.lang.String text)
encodingType - text - public static java.lang.String decode(com.knuddels.jtokkit.api.EncodingType encodingType,
java.util.List<java.lang.Integer> encoded)
encodingType - encoded - public static com.knuddels.jtokkit.api.Encoding getEncoding(java.lang.String modelName)
modelName - public static java.util.List<java.lang.Integer> encode(java.lang.String modelName,
java.lang.String text)
text - Text informationpublic static int tokens(java.lang.String modelName,
java.lang.String text)
modelName - text - public static int tokens(java.lang.String modelName,
java.util.List<ChatMessage> messages)
modelName - messages - public static java.lang.String decode(java.lang.String modelName,
java.util.List<java.lang.Integer> encoded)
modelName - encoded - public static com.knuddels.jtokkit.api.ModelType getModelTypeByName(java.lang.String name)
name - public static boolean isBlankChar(int c)
public static boolean isBlankChar(char c)
public static boolean isNotBlank(java.lang.CharSequence str)
public static boolean isBlank(java.lang.CharSequence str)