Provide type hints for pass-through Langchain and Llama-index objects (#95)
This commit is contained in:
committed by
GitHub
parent
e34b1e4c6d
commit
0ce3a8832f
@@ -8,6 +8,20 @@ class BaseSplitter(DocTransformer):
|
||||
|
||||
|
||||
class TokenSplitter(LlamaIndexDocTransformerMixin, BaseSplitter):
|
||||
def __init__(
|
||||
self,
|
||||
chunk_size: int = 1024,
|
||||
chunk_overlap: int = 20,
|
||||
separator: str = " ",
|
||||
**params,
|
||||
):
|
||||
super().__init__(
|
||||
chunk_size=chunk_size,
|
||||
chunk_overlap=chunk_overlap,
|
||||
separator=separator,
|
||||
**params,
|
||||
)
|
||||
|
||||
def _get_li_class(self):
|
||||
from llama_index.text_splitter import TokenTextSplitter
|
||||
|
||||
@@ -15,6 +29,9 @@ class TokenSplitter(LlamaIndexDocTransformerMixin, BaseSplitter):
|
||||
|
||||
|
||||
class SentenceWindowSplitter(LlamaIndexDocTransformerMixin, BaseSplitter):
|
||||
def __init__(self, window_size: int = 3, **params):
|
||||
super().__init__(window_size=window_size, **params)
|
||||
|
||||
def _get_li_class(self):
|
||||
from llama_index.node_parser import SentenceWindowNodeParser
|
||||
|
||||
|
Reference in New Issue
Block a user