37c744b616
* Modify docstore and vectorstore objects to be reconstructable * Simplify the file docstore * Use the simple file docstore and vector store in MVP
81 lines
2.4 KiB
Python
81 lines
2.4 KiB
Python
import json
|
|
from pathlib import Path
|
|
from typing import List, Optional, Union
|
|
|
|
from kotaemon.base import Document
|
|
|
|
from .base import BaseDocumentStore
|
|
|
|
|
|
class InMemoryDocumentStore(BaseDocumentStore):
|
|
"""Simple memory document store that store document in a dictionary"""
|
|
|
|
def __init__(self):
|
|
self._store = {}
|
|
|
|
def add(
|
|
self,
|
|
docs: Union[Document, List[Document]],
|
|
ids: Optional[Union[List[str], str]] = None,
|
|
**kwargs,
|
|
):
|
|
"""Add document into document store
|
|
|
|
Args:
|
|
docs: list of documents to add
|
|
ids: specify the ids of documents to add or
|
|
use existing doc.doc_id
|
|
exist_ok: raise error when duplicate doc-id
|
|
found in the docstore (default to False)
|
|
"""
|
|
exist_ok: bool = kwargs.pop("exist_ok", False)
|
|
|
|
if ids and not isinstance(ids, list):
|
|
ids = [ids]
|
|
if not isinstance(docs, list):
|
|
docs = [docs]
|
|
doc_ids = ids if ids else [doc.doc_id for doc in docs]
|
|
|
|
for doc_id, doc in zip(doc_ids, docs):
|
|
if doc_id in self._store and not exist_ok:
|
|
raise ValueError(f"Document with id {doc_id} already exist")
|
|
self._store[doc_id] = doc
|
|
|
|
def get(self, ids: Union[List[str], str]) -> List[Document]:
|
|
"""Get document by id"""
|
|
if not isinstance(ids, list):
|
|
ids = [ids]
|
|
|
|
return [self._store[doc_id] for doc_id in ids]
|
|
|
|
def get_all(self) -> List[Document]:
|
|
"""Get all documents"""
|
|
return list(self._store.values())
|
|
|
|
def count(self) -> int:
|
|
"""Count number of documents"""
|
|
return len(self._store)
|
|
|
|
def delete(self, ids: Union[List[str], str]):
|
|
"""Delete document by id"""
|
|
if not isinstance(ids, list):
|
|
ids = [ids]
|
|
|
|
for doc_id in ids:
|
|
del self._store[doc_id]
|
|
|
|
def save(self, path: Union[str, Path]):
|
|
"""Save document to path"""
|
|
store = {key: value.to_dict() for key, value in self._store.items()}
|
|
with open(path, "w") as f:
|
|
json.dump(store, f)
|
|
|
|
def load(self, path: Union[str, Path]):
|
|
"""Load document store from path"""
|
|
with open(path) as f:
|
|
store = json.load(f)
|
|
self._store = {key: Document.from_dict(value) for key, value in store.items()}
|
|
|
|
def __persist_flow__(self):
|
|
return {}
|