import py_vncorenlp
from utils import get_data_from_yaml

config = get_data_from_yaml("config.yaml")
VNCORENLP_DIR = config.get("vncorenlp")["save_dir"]


# Load the word and sentence segmentation component
rdrsegmenter = py_vncorenlp.VnCoreNLP(annotators=['wseg'], save_dir=VNCORENLP_DIR)

def preprocess_text(text):
    content = rdrsegmenter.word_segment(text)
    content = ''.join(content)
    return content