Public/kotaemon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

allow chunk_overlap with zero value (#457) #none

Browse Source 
Co-authored-by: Tadashi <tadashi@cinnamon.is>
This commit is contained in:
Pedro Lima 2024-11-05 07:12:52 +00:00 committed by GitHub
parent d127fec9f7
commit 908452cc18
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
libs/ktem/ktem/index/file/pipelines.py
View File

@ -742,7 +742,7 @@ class IndexDocumentPipeline(BaseFileIndexIndexing):
loader=reader, loader=reader,
splitter=TokenSplitter( splitter=TokenSplitter(
chunk_size=chunk_size or 1024, chunk_size=chunk_size or 1024,
chunk_overlap=chunk_overlap or 256, chunk_overlap=chunk_overlap if chunk_overlap is not None else 256,
separator="\n\n", separator="\n\n",
backup_separators=["\n", ".", "\u200B"], backup_separators=["\n", ".", "\u200B"],
), ),