From 074123b9b9fa2aa56aee457e9ef1e42639cb8c13 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Fri, 22 Dec 2023 16:22:42 +0530 Subject: [PATCH] Merge cloud, local dockerize workflows - Delete unused config directory --- .github/workflows/dockerize.yml | 49 ++++++++++++++++-- .github/workflows/dockerize_production.yml | 60 ---------------------- config/khoj_docker.yml | 51 ------------------ config/khoj_sample.yml | 57 -------------------- 4 files changed, 44 insertions(+), 173 deletions(-) delete mode 100644 .github/workflows/dockerize_production.yml delete mode 100644 config/khoj_docker.yml delete mode 100644 config/khoj_sample.yml diff --git a/.github/workflows/dockerize.yml b/.github/workflows/dockerize.yml index bc2da688..b9673e97 100644 --- a/.github/workflows/dockerize.yml +++ b/.github/workflows/dockerize.yml @@ -8,24 +8,47 @@ on: - master paths: - src/khoj/** - - config/** - pyproject.toml - Dockerfile + - prod.Dockerfile - docker-compose.yml - .github/workflows/dockerize.yml workflow_dispatch: + inputs: + tag: + description: 'Docker image tag' + default: 'dev' + khoj: + description: 'Build Khoj docker image' + type: boolean + default: true + khoj-cloud: + description: 'Build Khoj cloud docker image' + type: boolean + default: true env: - # Tag Image with tag name on release, else with 'pre' if push to master - DOCKER_IMAGE_TAG: ${{ github.ref_type == 'tag' && github.ref_name || 'pre' }} + # Tag Image with tag name on release + # else with user specified tag (default 'dev') if triggered via workflow + # else with 'pre' (if push to master) + DOCKER_IMAGE_TAG: ${{ github.ref_type == 'tag' && github.ref_name || github.event_name == 'workflow_dispatch' && github.event.inputs.tag || 'pre' }} jobs: build: - name: Build Docker Image, Push to Container Registry + name: Publish Khoj Docker Images runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + image: + - 'local' + - 'cloud' steps: - name: Checkout Code uses: actions/checkout@v3 + with: + # Get all history to correctly infer Khoj version using hatch + fetch-depth: 0 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -43,6 +66,7 @@ jobs: - name: 📦 Build and Push Docker Image uses: docker/build-push-action@v2 + if: (matrix.image == 'local' && github.event_name == 'workflow_dispatch') && github.event.inputs.khoj == 'true' || (matrix.image == 'local' && github.event_name == 'push') with: context: . file: Dockerfile @@ -50,7 +74,22 @@ jobs: push: true tags: | ghcr.io/${{ github.repository }}:${{ env.DOCKER_IMAGE_TAG }} - ${{ github.ref_type == 'tag' && 'ghcr.io/${{ github.repository }}-cloud:latest' || '' }} + ${{ github.ref_type == 'tag' && format('ghcr.io/{0}:latest', github.repository) || '' }} + build-args: | + VERSION=${{ steps.hatch.outputs.version }} + PORT=42110 + + - name: 📦️⛅️ Build and Push Cloud Docker Image + uses: docker/build-push-action@v2 + if: (matrix.image == 'cloud' && github.event_name == 'workflow_dispatch') && github.event.inputs.khoj-cloud == 'true' || (matrix.image == 'cloud' && github.event_name == 'push') + with: + context: . + file: prod.Dockerfile + platforms: linux/amd64 + push: true + tags: | + ghcr.io/${{ github.repository }}-cloud:${{ env.DOCKER_IMAGE_TAG }} + ${{ github.ref_type == 'tag' && format('ghcr.io/{0}-cloud:latest', github.repository) || '' }} build-args: | VERSION=${{ steps.hatch.outputs.version }} PORT=42110 diff --git a/.github/workflows/dockerize_production.yml b/.github/workflows/dockerize_production.yml deleted file mode 100644 index a42aa8d9..00000000 --- a/.github/workflows/dockerize_production.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: dockerize production - -on: - pull_request: - paths: - - src/khoj/** - - pyproject.toml - - prod.Dockerfile - - .github/workflows/dockerize_production.yml - push: - tags: - - "*" - branches: - - master - paths: - - src/khoj/** - - pyproject.toml - - prod.Dockerfile - - .github/workflows/dockerize_production.yml - workflow_dispatch: - -env: - # Tag Image with tag name on release, else with 'pre' if push to master else with 'dev' if push to PR branch - DOCKER_IMAGE_TAG: ${{ github.ref_type == 'tag' && github.ref_name || github.event_name == 'pull_request' && 'dev' || 'pre' }} - -jobs: - build: - name: Build Production Docker Image, Push to Container Registry - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.PAT }} - - - name: Get App Version - id: hatch - run: echo "::set-output name=version::$(pipx run hatch version)" - - - name: 📦 Build and Push Docker Image - uses: docker/build-push-action@v2 - with: - context: . - file: prod.Dockerfile - platforms: linux/amd64 - push: true - tags: | - ghcr.io/${{ github.repository }}-cloud:${{ env.DOCKER_IMAGE_TAG }} - ${{ github.ref_type == 'tag' && 'ghcr.io/${{ github.repository }}-cloud:latest' || '' }} - build-args: | - VERSION=${{ steps.hatch.outputs.version }} - PORT=42110 diff --git a/config/khoj_docker.yml b/config/khoj_docker.yml deleted file mode 100644 index 5fb89665..00000000 --- a/config/khoj_docker.yml +++ /dev/null @@ -1,51 +0,0 @@ -content-type: - # The /data/folder/ prefix to the folders is here because this is - # the directory to which the local files are copied in the docker-compose. - # If changing, the docker-compose volumes should also be changed to match. - org: - input-files: null - input-filter: ["/data/org/**/*.org"] - compressed-jsonl: "/data/embeddings/notes.jsonl.gz" - embeddings-file: "/data/embeddings/note_embeddings.pt" - index_heading_entries: false - - markdown: - input-files: null - input-filter: ["/data/markdown/**/*.markdown"] - compressed-jsonl: "/data/embeddings/markdown.jsonl.gz" - embeddings-file: "/data/embeddings/markdown_embeddings.pt" - - pdf: - input-files: null - input-filter: ["/data/pdf/**/*.pdf"] - compressed-jsonl: "/data/embeddings/pdf.jsonl.gz" - embeddings-file: "/data/embeddings/pdf_embeddings.pt" - - image: - input-directories: ["/data/images/"] - embeddings-file: "/data/embeddings/image_embeddings.pt" - batch-size: 50 - use-xmp-metadata: false - - notion: null - github: null - plugins: null - -search-type: - symmetric: null - asymmetric: - encoder: "sentence-transformers/multi-qa-MiniLM-L6-cos-v1" - cross-encoder: "cross-encoder/ms-marco-MiniLM-L-6-v2" - model_directory: "/data/models/asymmetric" - image: - encoder: "sentence-transformers/clip-ViT-B-32" - model_directory: "/data/models/image_encoder" - -processor: - conversation: - conversation-logfile: "/data/embeddings/conversation_logs.json" - enable-offline-chat: false - openai: null - -app: - should_log_telemetry: true diff --git a/config/khoj_sample.yml b/config/khoj_sample.yml deleted file mode 100644 index a30b02d9..00000000 --- a/config/khoj_sample.yml +++ /dev/null @@ -1,57 +0,0 @@ -content-type: - org: - input-files: # ["/path/to/org-file.org"] REQUIRED IF input-filter IS NOT SET OR - input-filter: # ["/path/to/org/*.org"] REQUIRED IF input-files IS NOT SET - compressed-jsonl: "~/.khoj/content/org/org.jsonl.gz" - embeddings-file: "~/.khoj/content/org/org_embeddings.pt" - index_heading_entries: false # Set to true to index entries with empty body - - markdown: - input-files: # ["/path/to/markdown-file.md"] REQUIRED IF input-filter IS NOT SET OR - input-filter: # ["/path/to/markdown/*.md"] REQUIRED IF input-files IS NOT SET - compressed-jsonl: "~/.khoj/content/markdown/markdown.jsonl.gz" - embeddings-file: "~/.khoj/content/markdown/markdown_embeddings.pt" - - ledger: - input-files: # ["/path/to/ledger-file.beancount"] REQUIRED IF input-filter is not set OR - input-filter: # ["/path/to/ledger/*.beancount"] REQUIRED IF input-files is not set - compressed-jsonl: "~/.khoj/content/ledger/ledger.jsonl.gz" - embeddings-file: "~/.khoj/content/ledger/ledger_embeddings.pt" - - image: - input-directories: # ["/path/to/images/"] REQUIRED IF input-filter IS NOT SET OR - input-filter: # ["/path/to/images/*.jpg"] REQUIRED IF input-directories IS NOT SET - embeddings-file: "~/.khoj/content/image/image_embeddings.pt" - batch-size: 50 - use-xmp-metadata: false - - music: - input-files: # ["/path/to/music-file.org"] REQUIRED IF input-filter IS NOT SET OR - input-filter: # ["/path/to/music/*.org"] REQUIRED IF input-files IS NOT SET - compressed-jsonl: "~/.khoj/content/music/music.jsonl.gz" - embeddings-file: "~/.khoj/content/music/music_embeddings.pt" - -search-type: - symmetric: - encoder: "sentence-transformers/all-MiniLM-L6-v2" - cross-encoder: "cross-encoder/ms-marco-MiniLM-L-6-v2" - encoder-type: sentence_transformers.SentenceTransformer - model_directory: "~/.khoj/search/symmetric/" - - asymmetric: - encoder: "sentence-transformers/multi-qa-MiniLM-L6-cos-v1" - cross-encoder: "cross-encoder/ms-marco-MiniLM-L-6-v2" - encoder-type: sentence_transformers.SentenceTransformer - model_directory: "~/.khoj/search/asymmetric/" - - image: - encoder: "sentence-transformers/clip-ViT-B-32" - encoder-type: sentence_transformers.SentenceTransformer - model_directory: "~/.khoj/search/image/" - -processor: - conversation: - openai-api-key: # "YOUR_OPENAI_API_KEY" - model: "text-davinci-003" - chat-model: "gpt-3.5-turbo" - conversation-logfile: "~/.khoj/processor/conversation/conversation_logs.json"