feat: add markdown file support (#202)

* feat: add support for .md

* fix: disable download all on private collection

flowsettings.py

@@ -222,7 +222,7 @@ KH_INDICES = [
         "config": {
             "supported_file_types": (
                 ".png, .jpeg, .jpg, .tiff, .tif, .pdf, .xls, .xlsx, .doc, .docx, "
-                ".pptx, .csv, .html, .mhtml, .txt, .zip"
+                ".pptx, .csv, .html, .mhtml, .txt, .md, .zip"
             ),
             "private": False,
         },
@@ -233,7 +233,7 @@ KH_INDICES = [
         "config": {
             "supported_file_types": (
                 ".png, .jpeg, .jpg, .tiff, .tif, .pdf, .xls, .xlsx, .doc, .docx, "
-                ".pptx, .csv, .html, .mhtml, .txt, .zip"
+                ".pptx, .csv, .html, .mhtml, .txt, .md, .zip"
             ),
             "private": False,
         },

libs/kotaemon/kotaemon/indices/ingests/files.py

@@ -18,6 +18,7 @@ from kotaemon.loaders import (
     OCRReader,
     PandasExcelReader,
     PDFThumbnailReader,
+    TxtReader,
     UnstructuredReader,
 )
 
@@ -47,6 +48,8 @@ KH_DEFAULT_FILE_EXTRACTORS: dict[str, BaseReader] = {
     ".tiff": unstructured,
     ".tif": unstructured,
     ".pdf": PDFThumbnailReader(),
+    ".txt": TxtReader(),
+    ".md": TxtReader(),
 }
 
 

libs/ktem/ktem/index/file/ui.py

@@ -353,6 +353,9 @@ class FileIndexPage(BasePage):
         return not is_zipped_state, new_button
 
     def download_all_files(self):
+        if self._index.config.get("private", False):
+            raise gr.Error("This feature is not available for private collection.")
+
         zip_files = []
         for file_name in os.listdir(flowsettings.KH_CHUNKS_OUTPUT_DIR):
             zip_files.append(os.path.join(flowsettings.KH_CHUNKS_OUTPUT_DIR, file_name))