From 654501e01c17f61f75326dcdf38002f3b5786a77 Mon Sep 17 00:00:00 2001 From: ian_Cin Date: Wed, 15 May 2024 16:34:50 +0700 Subject: [PATCH] (bump:minor) Feat: Add mechanism for user-site update and auto creating releases (#56) * move flowsettings.py and launch.py to root * update docs * sync sub package versions * rename launch.py to app.py and make run scripts work with installation package * add update scripts * auto version for root package * rename authors and update doc dir * Update auto-bump-and-release.yaml to trigger on push to main branch * latest as branch instead of tag * pin deps versions * cache the changelogs --- .github/workflows/auto-bump-and-release.yaml | 47 ++++++++++ .github/workflows/style-check.yaml | 2 +- .github/workflows/unit-test.yaml | 13 +-- README.md | 18 +++- libs/ktem/launch.py => app.py | 0 docs/about.md | 11 +++ docs/index.md | 2 +- docs/pages/app/customize-flows.md | 2 +- libs/ktem/flowsettings.py => flowsettings.py | 6 ++ libs/kotaemon/pyproject.toml | 89 +++++++++--------- libs/ktem/README.md | 24 ----- libs/ktem/ktem/pages/help.py | 84 ++++++++++++++--- libs/ktem/pyproject.toml | 40 ++++---- pyproject.toml | 27 +++++- scripts/run_linux.sh | 34 +++++-- scripts/run_macos.sh | 34 +++++-- scripts/run_windows.bat | 35 ++++--- scripts/update_linux.sh | 87 +++++++++++++++++ scripts/update_macos.sh | 87 +++++++++++++++++ scripts/update_windows.bat | 99 ++++++++++++++++++++ 20 files changed, 596 insertions(+), 145 deletions(-) create mode 100644 .github/workflows/auto-bump-and-release.yaml rename libs/ktem/launch.py => app.py (100%) create mode 100644 docs/about.md rename libs/ktem/flowsettings.py => flowsettings.py (97%) delete mode 100644 libs/ktem/README.md create mode 100644 scripts/update_linux.sh create mode 100644 scripts/update_macos.sh create mode 100644 scripts/update_windows.bat diff --git a/.github/workflows/auto-bump-and-release.yaml b/.github/workflows/auto-bump-and-release.yaml new file mode 100644 index 0000000..51f08c6 --- /dev/null +++ b/.github/workflows/auto-bump-and-release.yaml @@ -0,0 +1,47 @@ +name: Auto Bump and Release + +on: + push: + branches: + - main + +jobs: + auto-bump-and-release: + runs-on: ubuntu-latest + steps: + - name: Clone the repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Update Application Version + id: update-version + uses: anothrNick/github-tag-action@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WITH_V: true + DEFAULT_BUMP: none + MAJOR_STRING_TOKEN: "bump:major" + MINOR_STRING_TOKEN: "bump:minor" + PATCH_STRING_TOKEN: "bump:patch" + - name: Create release for ${{ steps.update-version.outputs.new_tag }} + run: | + echo Create release folder + mkdir kotaemon-app + echo ${{ steps.update-version.outputs.new_tag }} > kotaemon-app/VERSION + cp LICENSE.txt kotaemon-app/ + cp flowsettings.py kotaemon-app/ + cp app.py kotaemon-app/ + cp -r scripts kotaemon-app/ + tree kotaemon-app + zip -r kotaemon-app.zip kotaemon-app + - name: Release ${{ steps.update-version.outputs.new_tag }} + uses: softprops/action-gh-release@v2 + with: + files: kotaemon-app.zip + fail_on_unmatched_files: true + token: ${{ secrets.GITHUB_TOKEN }} + generate_release_notes: true + tag_name: ${{ steps.update-version.outputs.new_tag }} + make_latest: true + - name: Update latest branch + run: git branch -f latest tags/${{ steps.update-version.outputs.new_tag }} diff --git a/.github/workflows/style-check.yaml b/.github/workflows/style-check.yaml index 2047461..1e17041 100644 --- a/.github/workflows/style-check.yaml +++ b/.github/workflows/style-check.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Clone the repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup python uses: actions/setup-python@v4 with: diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index f49a79a..315acd8 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -33,7 +33,7 @@ jobs: name: unit testing with python ${{ matrix.python-version }} steps: - name: Clone the repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} @@ -56,11 +56,9 @@ jobs: - name: Get cache key id: get-cache-key - # using tomli so that it works on windows. From python 3.11, this can be switched to the - # built-in tomllib run: | - pip install tomli - package_version=$(python -c "import tomli; print(tomli.load(open('libs/kotaemon/pyproject.toml', 'rb'))['project']['version'])") + pip install "setuptools-git-versioning>=2.0,<3" + package_version=$(setuptools-git-versioning) cache_key="${{ runner.os }}-py${{ matrix.python-version }}-v${package_version}" echo "key=$cache_key" | tee -a ${{ matrix.GITHUB_OUTPUT }} @@ -82,10 +80,7 @@ jobs: cache_hit=${{ steps.restore-dependencies.outputs.cache-primary-key == steps.restore-dependencies.outputs.cache-matched-key }} echo "check=$cache_hit" | tee -a ${{ matrix.GITHUB_OUTPUT }} - - name: Install dependencies if ignore caching or no cache hit - if: | - steps.check-ignore-cache.outputs.check == 'true' || - steps.check-cache-hit.outputs.check != 'true' + - name: Install additional dependencies (if any) run: | python -m pip install --upgrade pip cd libs/kotaemon diff --git a/README.md b/README.md index 78db80e..9a0d43b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # kotaemon -Build and use local RAG-based Question Answering (QA) applications. +An open-source tool for chatting with your documents. Built with both end users and +developers in mind. https://github.com/Cinnamon/kotaemon/assets/25688648/815ecf68-3a02-4914-a0dd-3f8ec7e75cd9 @@ -28,6 +29,21 @@ documents and developers who want to build their own QA pipeline. - See your RAG pipeline in action with the provided UI (built with Gradio). - Share your pipeline so that others can use it. +```yml ++----------------------------------------------------------------------------+ +| End users: Those who use apps built with `kotaemon`. | +| (You use an app like the one in the demo above) | +| +----------------------------------------------------------------+ | +| | Developers: Those who built with `kotaemon`. | | +| | (You have `import kotaemon` somewhere in your project) | | +| | +----------------------------------------------------+ | | +| | | Contributors: Those who make `kotaemon` better. | | | +| | | (You make PR to this repo) | | | +| | +----------------------------------------------------+ | | +| +----------------------------------------------------------------+ | ++----------------------------------------------------------------------------+ +``` + This repository is under active development. Feedback, issues, and PRs are highly appreciated. Your input is valuable as it helps us persuade our business guys to support open source. diff --git a/libs/ktem/launch.py b/app.py similarity index 100% rename from libs/ktem/launch.py rename to app.py diff --git a/docs/about.md b/docs/about.md new file mode 100644 index 0000000..4792b87 --- /dev/null +++ b/docs/about.md @@ -0,0 +1,11 @@ +# About Kotaemon + +An open-source tool for chatting with your documents. Built with both end users and +developers in mind. + +[Source Code](https://github.com/Cinnamon/kotaemon) | +[Live Demo](https://huggingface.co/spaces/lone17/kotaemon-app) + +[User Guide](https://cinnamon.github.io/kotaemon/) | +[Developer Guide](https://cinnamon.github.io/kotaemon/development/) | +[Feedback](https://github.com/Cinnamon/kotaemon/issues) diff --git a/docs/index.md b/docs/index.md index 4f8af20..134b4f5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -19,7 +19,7 @@ Download and upzip the latest version of `kotaemon` by clicking this 2. Enable All Applications and choose Terminal. 3. NOTE: If you always want to open that file with Terminal, then check Always Open With. 4. From now on, double click on your file and it should work. - - Linux: `run_linux.sh`. If you are using Linux, you would know how to run a bash script, right ? + - Linux: `run_linux.sh`. Please run the script using `bash run_linux.sh` in your terminal. 2. After the installation, the installer will ask to launch the ktem's UI, answer to continue. 3. If launched, the application will be open automatically in your browser. diff --git a/docs/pages/app/customize-flows.md b/docs/pages/app/customize-flows.md index 40d320f..6d8e4da 100644 --- a/docs/pages/app/customize-flows.md +++ b/docs/pages/app/customize-flows.md @@ -8,7 +8,7 @@ At high level, to add new indexing and reasoning pipeline: `BaseComponent`. 2. You declare that class in the setting files `flowsettings.py`. -Then when `python launch.py`, the application will dynamically load those +Then when `python app.py`, the application will dynamically load those pipelines. The below sections talk in more detail about how the pipelines should be diff --git a/libs/ktem/flowsettings.py b/flowsettings.py similarity index 97% rename from libs/ktem/flowsettings.py rename to flowsettings.py index 2ac0582..f4b5bbf 100644 --- a/libs/ktem/flowsettings.py +++ b/flowsettings.py @@ -11,6 +11,9 @@ if cur_frame is None: this_file = getframeinfo(cur_frame).filename this_dir = Path(this_file).parent +# change this if your app use a different name +KH_PACKAGE_NAME = "kotaemon_app" + # App can be ran from anywhere and it's not trivial to decide where to store app data. # So let's use the same directory as the flowsetting.py file. KH_APP_DATA_DIR = this_dir / "ktem_app_data" @@ -20,6 +23,9 @@ KH_APP_DATA_DIR.mkdir(parents=True, exist_ok=True) KH_USER_DATA_DIR = KH_APP_DATA_DIR / "user_data" KH_USER_DATA_DIR.mkdir(parents=True, exist_ok=True) +# doc directory +KH_DOC_DIR = this_dir / "docs" + # HF models can be big, let's store them in the app data directory so that it's easier # for users to manage their storage. # ref: https://huggingface.co/docs/huggingface_hub/en/guides/manage-cache diff --git a/libs/kotaemon/pyproject.toml b/libs/kotaemon/pyproject.toml index b218b04..6439eaa 100644 --- a/libs/kotaemon/pyproject.toml +++ b/libs/kotaemon/pyproject.toml @@ -1,6 +1,6 @@ # build backand and build dependencies [build-system] -requires = ["setuptools >= 61.0"] +requires = ["setuptools >= 61.0", "wheel", "setuptools-git-versioning>=2.0,<3"] build-backend = "setuptools.build_meta" [tool.setuptools] @@ -8,67 +8,71 @@ include-package-data = false packages.find.include = ["kotaemon*"] packages.find.exclude = ["tests*", "env*"] +[tool.setuptools-git-versioning] +enabled = true +dev_template = "{tag}" +dirty_template = "{tag}" +tag_filter = "v?\\d+(\\.\\d+)*.*" + # metadata and dependencies [project] name = "kotaemon" -version = "0.3.12" +dynamic = ["version"] requires-python = ">= 3.10" description = "Kotaemon core library for AI development." dependencies = [ - "langchain", - "langchain-community", - "langchain-openai", - "openai", - "theflow", - "llama-index>=0.9.0,<0.10.0", - "llama-hub", - "gradio>=4.26.0", - "openpyxl", - "cookiecutter", - "click", - "pandas", - "trogon", - "tenacity", - "python-dotenv", # currently used to read configs from file, should be remove in the future - "chromadb", - "unstructured", - "pypdf", - "html2text", - "fastembed", + "langchain>=0.1.16,<0.2.0", + "langchain-community>=0.0.34,<0.1.0", + "langchain-openai>=0.1.4,<0.2.0", + "openai>=1.23.6,<2", + "theflow>=0.8.6,<0.9.0", + "llama-index==0.9.48", + "llama-hub>=0.0.79,<0.1.0", + "gradio>=4.26.0,<5", + "openpyxl>=3.1.2,<3.2", + "cookiecutter>=2.6.0,<2.7", + "click>=8.1.7,<9", + "pandas>=2.2.2,<2.3", + "trogon>=0.5.0,<0.6", + "tenacity>=8.2.3,<8.3", + "python-dotenv>=1.0.1,<1.1", + "chromadb>=0.4.21,<0.5", + "unstructured==0.13.4", + "pypdf>=4.2.0,<4.3", + "html2text==2024.2.26", + "fastembed==0.2.6", + "llama-cpp-python==0.2.65", + "azure-ai-documentintelligence", + "cohere>=5.3.2,<5.4", ] readme = "README.md" -license = { text = "MIT License" } authors = [ - { name = "john", email = "john@cinnamon.is" }, - { name = "ian", email = "ian@cinnamon.is" }, - { name = "tadashi", email = "tadashi@cinnamon.is" }, + { name = "@trducng", email = "john@cinnamon.is" }, + { name = "@lone17", email = "ian@cinnamon.is" }, + { name = "@taprosoft", email = "tadashi@cinnamon.is" }, + { name = "@cin-albert", email = "albert@cinnamon.is" }, ] classifiers = [ "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ] [project.optional-dependencies] adv = [ - "wikipedia", - "duckduckgo-search", - "googlesearch-python", - "python-docx", - "pytest-mock", - "unstructured[pdf]", - "sentence_transformers", - "cohere", - "elasticsearch", - "llama-cpp-python", + "wikipedia>=1.4.0,<1.5", + "duckduckgo-search>=5.3.0,<5.4.0", + "googlesearch-python>=1.2.4,<1.3", + "python-docx>=1.1.0,<1.2", + "unstructured[pdf]==0.13.4", + "sentence_transformers==2.7.0", + "elasticsearch>=8.13.0,<8.14", "pdfservices-sdk @ git+https://github.com/niallcm/pdfservices-python-sdk.git@bump-and-unfreeze-requirements", - "fastembed", - "beautifulsoup4", - "azure-ai-documentintelligence", + "beautifulsoup4>=4.12.3,<4.13", ] dev = [ "ipython", "pytest", + "pytest-mock", "pre-commit", "black", "flake8", @@ -80,8 +84,3 @@ all = ["kotaemon[adv,dev]"] [project.scripts] kotaemon = "kotaemon.cli:main" - -[project.urls] -Homepage = "https://github.com/Cinnamon/kotaemon/" -Repository = "https://github.com/Cinnamon/kotaemon/" -Documentation = "https://github.com/Cinnamon/kotaemon/wiki" diff --git a/libs/ktem/README.md b/libs/ktem/README.md deleted file mode 100644 index 5af3ea9..0000000 --- a/libs/ktem/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# Example of MVP pipeline for _example_ - -## Prerequisite - -To run the system out-of-the-box, please supply the following environment -variables: - -``` -OPENAI_API_KEY= -OPENAI_API_BASE= -OPENAI_API_VERSION= -SERPAPI_API_KEY= -COHERE_API_KEY= -OPENAI_API_KEY_EMBEDDING= - -# optional -KH_APP_NAME= -``` - -## Run - -``` -gradio launch.py -``` diff --git a/libs/ktem/ktem/pages/help.py b/libs/ktem/ktem/pages/help.py index 3324939..54decb6 100644 --- a/libs/ktem/ktem/pages/help.py +++ b/libs/ktem/ktem/pages/help.py @@ -1,25 +1,83 @@ +from importlib.metadata import version from pathlib import Path import gradio as gr +import requests +from theflow.settings import settings + +CHANGELOG_CACHE_DIR = Path(settings.KH_APP_DATA_DIR) / "changelogs" + + +def get_remote_doc(url): + try: + res = requests.get(url) + return res.text + except Exception as e: + print(f"Failed to fetch document from {url}: {e}") + return "" + + +def get_changelogs(version): + # try retrieve from cache + if (CHANGELOG_CACHE_DIR / f"{version}.md").exists(): + with open(CHANGELOG_CACHE_DIR / f"{version}.md", "r") as fi: + return fi.read() + + release_url = f"https://api.github.com/repos/Cinnamon/kotaemon/releases/{version}" + try: + res = requests.get(release_url).json() + changelogs = res.get("body", "") + + # cache the changelogs + with open(CHANGELOG_CACHE_DIR / f"{version}.md", "w") as fi: + fi.write(changelogs) + + return changelogs + except Exception as e: + print(f"Failed to fetch changelogs from {release_url}: {e}") + return "" class HelpPage: def __init__(self, app): self._app = app - self.md_dir = Path(__file__).parent.parent / "assets" / "md" - self.doc_dir = Path(__file__).parents[4] / "docs" + self.doc_dir = Path(settings.KH_DOC_DIR) + self.remote_content_url = "https://raw.githubusercontent.com/Cinnamon/kotaemon" - with gr.Accordion("About"): - with (self.md_dir / "about.md").open(encoding="utf-8") as fi: - gr.Markdown(fi.read()) + self.app_version = None + try: + # Caution: This might produce the wrong version + # https://stackoverflow.com/a/59533071 + self.app_version = version(settings.KH_PACKAGE_NAME) + except Exception as e: + print(f"Failed to get app version: {e}") - with gr.Accordion("User Guide"): + about_md_dir = self.doc_dir / "about.md" + if about_md_dir.exists(): + with (self.doc_dir / "about.md").open(encoding="utf-8") as fi: + about_md = fi.read() + else: # fetch from remote + about_md = get_remote_doc( + f"{self.remote_content_url}/v{self.app_version}/docs/about.md" + ) + if about_md: + with gr.Accordion("About"): + gr.Markdown(about_md) + + user_guide_md_dir = self.doc_dir / "usage.md" + if user_guide_md_dir.exists(): with (self.doc_dir / "usage.md").open(encoding="utf-8") as fi: - gr.Markdown(fi.read()) + user_guide_md = fi.read() + else: # fetch from remote + user_guide_md = get_remote_doc( + f"{self.remote_content_url}/v{self.app_version}/docs/usage.md" + ) + if user_guide_md: + with gr.Accordion("User Guide"): + gr.Markdown(user_guide_md) - with gr.Accordion("Changelogs"): - gr.Markdown(self.get_changelogs()) - - def get_changelogs(self): - with (self.md_dir / "changelogs.md").open(encoding="utf-8") as fi: - return fi.read() + if self.app_version: + changelogs = get_changelogs("tags/v" + self.app_version) + if changelogs: + with gr.Accordion(f"Changelogs (v{self.app_version})"): + gr.Markdown(changelogs) diff --git a/libs/ktem/pyproject.toml b/libs/ktem/pyproject.toml index 1e66b3f..e50c63d 100644 --- a/libs/ktem/pyproject.toml +++ b/libs/ktem/pyproject.toml @@ -1,37 +1,41 @@ [build-system] -requires = ["setuptools >= 61.0"] +requires = ["setuptools >= 61.0", "wheel", "setuptools-git-versioning>=2.0,<3"] build-backend = "setuptools.build_meta" [tool.setuptools] include-package-data = true +packages.find.exclude = ["ktem_tests*", "env*"] packages.find.include = ["ktem*"] -packages.find.exclude = ["tests*", "env*"] + +[tool.setuptools-git-versioning] +enabled = true +dev_template = "{tag}" +dirty_template = "{tag}" +tag_filter = "v?\\d+(\\.\\d+)*.*" [project] name = "ktem" -version = "0.2.0" +dynamic = ["version"] requires-python = ">= 3.10" description = "RAG-based Question and Answering Application" dependencies = [ - "click", - "platformdirs", - "pluggy", - "python-decouple", - "sqlalchemy", - "sqlmodel", - "tiktoken", - "gradio>=4.26.0", - "markdown", + "click>=8.1.7,<9", + "platformdirs>=4.2.1,<5", + "pluggy>=1.5.0,<2", + "python-decouple>=3.8,<4", + "SQLAlchemy>=2.0.29,<3", + "sqlmodel>=0.0.16,<0.1", + "tiktoken>=0.6.0<1", + "gradio>=4.26.0,<5", + "markdown>=3.6,<4", ] -readme = "README.md" -license = { text = "MIT License" } authors = [ - { name = "john", email = "john@cinnamon.is" }, - { name = "ian", email = "ian@cinnamon.is" }, - { name = "tadashi", email = "tadashi@cinnamon.is" }, + { name = "@trducng", email = "john@cinnamon.is" }, + { name = "@lone17", email = "ian@cinnamon.is" }, + { name = "@taprosoft", email = "tadashi@cinnamon.is" }, + { name = "@cin-albert", email = "albert@cinnamon.is" }, ] classifiers = [ "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ] diff --git a/pyproject.toml b/pyproject.toml index cb02187..2e39c68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,20 +1,41 @@ [build-system] -requires = ["setuptools >= 61.0"] +requires = ["setuptools >= 61.0", "wheel", "setuptools-git-versioning>=2.0,<3"] build-backend = "setuptools.build_meta" [tool.setuptools] include-package-data = false packages.find.include = [] +[tool.setuptools-git-versioning] +enabled = true +dev_template = "{tag}" +dirty_template = "{tag}" +tag_filter = "v?\\d+(\\.\\d+)*.*" + [project] name = "kotaemon-app" -version = "0.0.1" +dynamic = ["version"] requires-python = ">= 3.10" description = "Kotaemon App" dependencies = [ - "kotaemon @ git+https://github.com/Cinnamon/kotaemon.git/@main#subdirectory=libs/kotaemon", + "kotaemon @ git+https://github.com/Cinnamon/kotaemon.git@main#subdirectory=libs/kotaemon", "ktem @ git+https://github.com/Cinnamon/kotaemon.git@main#subdirectory=libs/ktem" ] +authors = [ + { name = "@trducng", email = "john@cinnamon.is" }, + { name = "@lone17", email = "ian@cinnamon.is" }, + { name = "@taprosoft", email = "tadashi@cinnamon.is" }, + { name = "@cin-albert", email = "albert@cinnamon.is" }, +] +classifiers = [ + "Programming Language :: Python :: 3", + "Operating System :: OS Independent", +] + +[project.urls] +Homepage = "https://cinnamon.github.io/kotaemon/" +Repository = "https://github.com/Cinnamon/kotaemon/" +Documentation = "https://cinnamon.github.io/kotaemon/" [tool.codespell] skip = "*.js,*.css,*.map" diff --git a/scripts/run_linux.sh b/scripts/run_linux.sh index 774292a..8eff787 100755 --- a/scripts/run_linux.sh +++ b/scripts/run_linux.sh @@ -95,11 +95,27 @@ function install_dependencies() { local kotaemon_root="$(pwd)/libs/kotaemon" local ktem_root="$(pwd)/libs/ktem/" - echo "" && echo "Install kotaemon's requirements" - python -m pip install -e "$kotaemon_root" + if [ -f "$(pwd)/VERSION" ]; then + local app_version=$(<"$(pwd)/VERSION") + else + local app_version="latest" + fi - echo "" && echo "Install ktem's requirements" - python -m pip install -e "$ktem_root" + if [ -f "pyproject.toml" ]; then + echo "Found pyproject.toml. Installing from source" + echo "" && echo "Installing libs/kotaemon" + python -m pip install -e "$kotaemon_root" + echo "" && echo "Installing libs/ktem" + python -m pip install -e "$ktem_root" + + python -m pip install --no-deps -e . + else + echo "Installing Kotaemon $app_version" + # Work around for versioning control + python -m pip install "git+https://github.com/Cinnamon/kotaemon.git@$app_version#subdirectory=libs/kotaemon" + python -m pip install "git+https://github.com/Cinnamon/kotaemon.git@$app_version#subdirectory=libs/ktem" + python -m pip install --no-deps "git+https://github.com/Cinnamon/kotaemon.git@$app_version" + fi if ! pip list 2>/dev/null | grep -q "kotaemon"; then echo "Installation failed. You may need to run the installer again." @@ -128,7 +144,7 @@ function setup_local_model() { } function launch_ui() { - python $(pwd)/libs/ktem/launch.py || { + python $(pwd)/app.py || { echo "" && echo "Will exit now..." exit 1 } @@ -153,20 +169,20 @@ python_version="3.10" check_path_for_spaces -print_highlight "Setup Anaconda/Miniconda" +print_highlight "Setting up Miniconda" install_miniconda -print_highlight "Create and Activate conda environment" +print_highlight "Creating conda environment" create_conda_env "$python_version" activate_conda_env -print_highlight "Install requirements" +print_highlight "Installing requirements" install_dependencies print_highlight "Setting up a local model" setup_local_model -print_highlight "Launching web UI. Please wait..." +print_highlight "Launching Kotaemon in your browser, please wait..." launch_ui deactivate_conda_env diff --git a/scripts/run_macos.sh b/scripts/run_macos.sh index 4f2762f..011818f 100755 --- a/scripts/run_macos.sh +++ b/scripts/run_macos.sh @@ -95,11 +95,27 @@ function install_dependencies() { local kotaemon_root="$(pwd)/libs/kotaemon" local ktem_root="$(pwd)/libs/ktem/" - echo "" && echo "Install kotaemon's requirements" - python -m pip install -e "$kotaemon_root" + if [ -f "$(pwd)/VERSION" ]; then + local app_version=$(<"$(pwd)/VERSION") + else + local app_version="latest" + fi - echo "" && echo "Install ktem's requirements" - python -m pip install -e "$ktem_root" + if [ -f "pyproject.toml" ]; then + echo "Found pyproject.toml. Installing from source" + echo "" && echo "Installing libs/kotaemon" + python -m pip install -e "$kotaemon_root" + echo "" && echo "Installing libs/ktem" + python -m pip install -e "$ktem_root" + + python -m pip install --no-deps -e . + else + echo "Installing Kotaemon $app_version" + # Work around for versioning control + python -m pip install "git+https://github.com/Cinnamon/kotaemon.git@$app_version#subdirectory=libs/kotaemon" + python -m pip install "git+https://github.com/Cinnamon/kotaemon.git@$app_version#subdirectory=libs/ktem" + python -m pip install --no-deps "git+https://github.com/Cinnamon/kotaemon.git@$app_version" + fi if ! pip list 2>/dev/null | grep -q "kotaemon"; then echo "Installation failed. You may need to run the installer again." @@ -129,7 +145,7 @@ function setup_local_model() { } function launch_ui() { - python $(pwd)/libs/ktem/launch.py || { + python $(pwd)/app.py || { echo "" && echo "Will exit now..." exit 1 } @@ -157,20 +173,20 @@ python_version="3.10" check_path_for_spaces -print_highlight "Setup Anaconda/Miniconda" +print_highlight "Setting up Miniconda" install_miniconda -print_highlight "Create and Activate conda environment" +print_highlight "Creating conda environment" create_conda_env "$python_version" activate_conda_env -print_highlight "Install requirements" +print_highlight "Installing requirements" install_dependencies print_highlight "Setting up a local model" setup_local_model -print_highlight "Launching web UI. Please wait..." +print_highlight "Launching Kotaemon in your browser, please wait..." launch_ui deactivate_conda_env diff --git a/scripts/run_windows.bat b/scripts/run_windows.bat index 50050d7..369610d 100644 --- a/scripts/run_windows.bat +++ b/scripts/run_windows.bat @@ -3,6 +3,7 @@ :: Main script execution CD /D "%~dp0\.." +SET /p app_version=<"%CD%\VERSION" || SET app_version=latest SET install_dir=%CD%\install_dir SET conda_root=%install_dir%\conda SET env_dir=%install_dir%\env @@ -15,19 +16,19 @@ IF %ERRORLEVEL% EQU 0 ( GOTO :end ) -CALL :print_highlight "Setup Anaconda/Miniconda" +CALL :print_highlight "Setting up Miniconda" CALL :download_and_install_miniconda :: check if function run fail, then exit the script IF ERRORLEVEL 1 GOTO :end -CALL :print_highlight "Create and Activate conda environment" +CALL :print_highlight "Creating conda environment" CALL :create_conda_environment IF ERRORLEVEL 1 GOTO :end CALL :activate_environment IF ERRORLEVEL 1 GOTO :end -CALL :print_highlight "Install requirements" +CALL :print_highlight "Installing Kotaemon" CALL :install_dependencies IF ERRORLEVEL 1 GOTO :end @@ -35,7 +36,7 @@ CALL :print_highlight "Setting up a local model" CALL :setup_local_model IF ERRORLEVEL 1 GOTO :end -CALL :print_highlight "Launching web UI. Please wait..." +CALL :print_highlight "Launching Kotaemon in your browser, please wait..." CALL :launch_ui CALL :deactivate_environment @@ -113,11 +114,23 @@ pip list | findstr /C:"kotaemon" >NUL 2>&1 IF %ERRORLEVEL% == 0 ( ECHO Dependencies are already installed ) ELSE ( - ECHO Install kotaemon's requirements - CALL python -m pip install -e "%CD%\libs\kotaemon" + IF EXIST "pyproject.toml" ( + ECHO Found pyproject.toml. Installing from source... - ECHO Install ktem's requirements - CALL python -m pip install -e "%CD%\libs\ktem" + ECHO Installing libs\kotaemon + python -m pip install -e "%CD%\libs\kotaemon" + + ECHO Installing libs\ktem + python -m pip install -e "%CD%\libs\ktem" + + python -m pip install --no-deps -e . + ) ELSE ( + ECHO Installing Kotaemon %app_version% + @REM Work around for versioning control + python -m pip install git+https://github.com/Cinnamon/kotaemon.git@"%app_version%"#subdirectory=libs/kotaemon + python -m pip install git+https://github.com/Cinnamon/kotaemon.git@"%app_version%"#subdirectory=libs/ktem + python -m pip install --no-deps git+https://github.com/Cinnamon/kotaemon.git@"%app_version%" + ) ( CALL pip list | findstr /C:"kotaemon" >NUL 2>&1 ) || ( ECHO. && ECHO Installation failed. You may need to run the installer again. @@ -126,8 +139,8 @@ IF %ERRORLEVEL% == 0 ( ) CALL :print_highlight "Install successfully. Clear cache..." - CALL "%conda_root%\condabin\conda.bat" clean --all -y - CALL python -m pip cache purge + "%conda_root%\condabin\conda.bat" clean --all -y + python -m pip cache purge ) GOTO :eof @@ -136,7 +149,7 @@ python "%CD%\scripts\serve_local.py" GOTO :eof :launch_ui -CALL python "%CD%\libs\ktem\launch.py" || ( ECHO. && ECHO Will exit now... && GOTO :exit_func_with_error ) +CALL python "%CD%\app.py" || ( ECHO. && ECHO Will exit now... && GOTO :exit_func_with_error ) GOTO :eof :print_highlight diff --git a/scripts/update_linux.sh b/scripts/update_linux.sh new file mode 100644 index 0000000..af0603c --- /dev/null +++ b/scripts/update_linux.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +# functions for better code organization +function check_path_for_spaces() { + if [[ $PWD =~ \ ]]; then + echo "The current workdir has whitespace which can lead to unintended behaviour. Please modify your path and continue later." + exit 1 + fi +} + +function activate_conda_env() { + # deactivate the current env(s) to avoid conflicts + { conda deactivate && conda deactivate && conda deactivate; } 2>/dev/null + + # check if conda env is broken (because of interruption during creation) + if [ ! -f "$env_dir/bin/python" ]; then + echo "Conda environment appears to be broken. You may need to remove $env_dir and run the installer again." + exit 1 + fi + + source "$conda_root/etc/profile.d/conda.sh" # conda init + conda activate "$env_dir" || { + echo "Failed to activate environment. Please remove $env_dir and run the installer again" + exit 1 + } + echo "Activate conda environment at $CONDA_PREFIX" +} + +function deactivate_conda_env() { + # Conda deactivate if we are in the right env + if [ "$CONDA_PREFIX" == "$env_dir" ]; then + conda deactivate + echo "Deactivate conda environment at $env_dir" + fi +} + +function update_latest() { + current_version=$(pip list | awk '/kotaemon-app/ {print $2}') + echo "Current version $current_version" + + if [ -f "pyproject.toml" ]; then + echo "Source files detected. Please perform git pull manually." + deactivate_environment + exit 1 + else + echo "Installing version: $app_version" + # Work around for versioning control + python -m pip install "git+https://github.com/Cinnamon/kotaemon.git@$app_version#subdirectory=libs/kotaemon" + python -m pip install "git+https://github.com/Cinnamon/kotaemon.git@$app_version#subdirectory=libs/ktem" + python -m pip install --no-deps git+https://github.com/Cinnamon/kotaemon.git@$app_version + if [ $? -ne 0 ]; then + echo + echo "Update failed. You may need to run the update again." + deactivate_environment + exit 1 + fi + fi +} + +function print_highlight() { + local message="${1}" + echo "" && echo "******************************************************" + echo $message + echo "******************************************************" && echo "" +} + +# Main script execution + +# move two levels up from the dir where this script resides +cd "$(dirname "${BASH_SOURCE[0]}")" && cd .. + +app_version="latest" +install_dir="$(pwd)/install_dir" +conda_root="${install_dir}/conda" +env_dir="${install_dir}/env" + +check_path_for_spaces + +print_highlight "Activating conda environment" +activate_conda_env + +print_highlight "Updating Kotaemon to latest" +update_latest + +deactivate_conda_env + +read -p "Press enter to continue" diff --git a/scripts/update_macos.sh b/scripts/update_macos.sh new file mode 100644 index 0000000..af0603c --- /dev/null +++ b/scripts/update_macos.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +# functions for better code organization +function check_path_for_spaces() { + if [[ $PWD =~ \ ]]; then + echo "The current workdir has whitespace which can lead to unintended behaviour. Please modify your path and continue later." + exit 1 + fi +} + +function activate_conda_env() { + # deactivate the current env(s) to avoid conflicts + { conda deactivate && conda deactivate && conda deactivate; } 2>/dev/null + + # check if conda env is broken (because of interruption during creation) + if [ ! -f "$env_dir/bin/python" ]; then + echo "Conda environment appears to be broken. You may need to remove $env_dir and run the installer again." + exit 1 + fi + + source "$conda_root/etc/profile.d/conda.sh" # conda init + conda activate "$env_dir" || { + echo "Failed to activate environment. Please remove $env_dir and run the installer again" + exit 1 + } + echo "Activate conda environment at $CONDA_PREFIX" +} + +function deactivate_conda_env() { + # Conda deactivate if we are in the right env + if [ "$CONDA_PREFIX" == "$env_dir" ]; then + conda deactivate + echo "Deactivate conda environment at $env_dir" + fi +} + +function update_latest() { + current_version=$(pip list | awk '/kotaemon-app/ {print $2}') + echo "Current version $current_version" + + if [ -f "pyproject.toml" ]; then + echo "Source files detected. Please perform git pull manually." + deactivate_environment + exit 1 + else + echo "Installing version: $app_version" + # Work around for versioning control + python -m pip install "git+https://github.com/Cinnamon/kotaemon.git@$app_version#subdirectory=libs/kotaemon" + python -m pip install "git+https://github.com/Cinnamon/kotaemon.git@$app_version#subdirectory=libs/ktem" + python -m pip install --no-deps git+https://github.com/Cinnamon/kotaemon.git@$app_version + if [ $? -ne 0 ]; then + echo + echo "Update failed. You may need to run the update again." + deactivate_environment + exit 1 + fi + fi +} + +function print_highlight() { + local message="${1}" + echo "" && echo "******************************************************" + echo $message + echo "******************************************************" && echo "" +} + +# Main script execution + +# move two levels up from the dir where this script resides +cd "$(dirname "${BASH_SOURCE[0]}")" && cd .. + +app_version="latest" +install_dir="$(pwd)/install_dir" +conda_root="${install_dir}/conda" +env_dir="${install_dir}/env" + +check_path_for_spaces + +print_highlight "Activating conda environment" +activate_conda_env + +print_highlight "Updating Kotaemon to latest" +update_latest + +deactivate_conda_env + +read -p "Press enter to continue" diff --git a/scripts/update_windows.bat b/scripts/update_windows.bat new file mode 100644 index 0000000..3ed2e67 --- /dev/null +++ b/scripts/update_windows.bat @@ -0,0 +1,99 @@ +@ECHO off + +:: Main script execution +CD /D "%~dp0\.." + +SET app_version=latest +SET install_dir=%CD%\install_dir +SET conda_root=%install_dir%\conda +SET env_dir=%install_dir%\env + +ECHO %CD%| FINDSTR /C:" " >nul 2>&1 +IF %ERRORLEVEL% EQU 0 ( + ECHO The current workdir has whitespace which can lead to unintended behaviour. Please modify your path and continue later. + GOTO :end +) + +CALL :print_highlight "Activating conda environment" +CALL :activate_environment +IF ERRORLEVEL 1 GOTO :end + +CALL :print_highlight "Updating Kotaemon to latest" +CALL :update_latest +IF ERRORLEVEL 1 GOTO :end + +CALL :deactivate_environment +GOTO :end_success + + +:activate_environment +:: deactivate existing conda env(s) to avoid conflicts +( CALL conda deactivate && CALL conda deactivate && CALL conda deactivate ) 2> nul + +CALL "%env_dir%\python.exe" --version >nul 2>&1 || ( + ECHO The environment appears to be broken. You may need to remove %env_dir% and run the installer again. + GOTO :exit_func_with_error +) + +CALL "%conda_root%\condabin\conda.bat" activate %env_dir% || ( + ECHO Failed to activate environment. You may need to remove %env_dir% and run the installer again. + GOTO :exit_func_with_error +) +ECHO Activate conda environment at %env_dir% + +GOTO :eof + +:deactivate_environment +:: Conda deactivate if we are in the right env +IF "%CONDA_PREFIX%" == "%env_dir%" ( + CALL "%conda_root%\condabin\conda.bat" deactivate + ECHO Deactivate conda environment at %env_dir% +) +GOTO :eof + +:update_latest +FOR /F "tokens=1,2" %%a in ('pip list') do if "%%a"=="kotaemon-app" set current_version=%%b +ECHO Current version %current_version% + +IF EXIST "pyproject.toml" ( + ECHO Source files detected. Please perform git pull manually. + CALL :deactivate_environment + GOTO :exit_func_with_error +) ELSE ( + ECHO Installing version: %app_version% + @REM Work around for versioning control + python -m pip install git+https://github.com/Cinnamon/kotaemon.git@"%app_version%"#subdirectory=libs/kotaemon + python -m pip install git+https://github.com/Cinnamon/kotaemon.git@"%app_version%"#subdirectory=libs/ktem + python -m pip install --no-deps git+https://github.com/Cinnamon/kotaemon.git@"%app_version%" +) || ( + ECHO. && ECHO Update failed. You may need to run the update again. + CALL :deactivate_environment + GOTO :exit_func_with_error +) + +CALL :print_highlight "Update successfully." +FOR /F "tokens=1,2" %%a in ('pip list') do if "%%a"=="kotaemon-app" set updated_version=%%b +ECHO Updated version %updated_version% +ECHO %updated_version% > VERSION +GOTO :eof + +:print_highlight +ECHO. && ECHO ****************************************************** +ECHO %~1 +ECHO ****************************************************** && ECHO. +GOTO :eof + +:exit_func_with_error +:: Called inside functions when error happens, then back to the main routine with error code 1 +EXIT /B 1 + +:end_success +:: Exit the script main routine with error code 0 (success) +ECHO Script completed successfully. +PAUSE +EXIT /B 0 + +:end +:: Exit the script main routine with error code 1 (fail) +PAUSE +EXIT /B 1