Package edu.tufts.hrilab.llm

Class Tokenizer

java.lang.Object

edu.tufts.hrilab.llm.Tokenizer

public class Tokenizer
extends Object

Constructor Summary

Constructors

Constructor	Description
Tokenizer()

Method Summary

Modifier and Type	Method	Description
Integer	count(String text)
List<Integer>	encode(String text)
void	setEncoding(String service, String model)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Details

Tokenizer

public Tokenizer()

Method Details

setEncoding

public void setEncoding(String service,
 String model)

encode

public List<Integer> encode(String text)

count

public Integer count(String text)