Merge cloud, local dockerize workflows

- Delete unused config directory

.github/workflows/dockerize.yml

@@ -8,24 +8,47 @@ on:
       - master
     paths:
       - src/khoj/**
-      - config/**
       - pyproject.toml
       - Dockerfile
+      - prod.Dockerfile
       - docker-compose.yml
       - .github/workflows/dockerize.yml
   workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Docker image tag'
+        default: 'dev'
+      khoj:
+        description: 'Build Khoj docker image'
+        type: boolean
+        default: true
+      khoj-cloud:
+        description: 'Build Khoj cloud docker image'
+        type: boolean
+        default: true
 
 env:
-  # Tag Image with tag name on release, else with 'pre' if push to master
+  # Tag Image with tag name on release
-  DOCKER_IMAGE_TAG: ${{ github.ref_type == 'tag' && github.ref_name || 'pre' }}
+  # else with user specified tag (default 'dev') if triggered via workflow
+  # else with 'pre' (if push to master)
+  DOCKER_IMAGE_TAG: ${{ github.ref_type == 'tag' && github.ref_name || github.event_name == 'workflow_dispatch' && github.event.inputs.tag || 'pre' }}
 
 jobs:
   build:
-    name: Build Docker Image, Push to Container Registry
+    name: Publish Khoj Docker Images
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        image:
+          - 'local'
+          - 'cloud'
     steps:
       - name: Checkout Code
         uses: actions/checkout@v3
+        with:
+          # Get all history to correctly infer Khoj version using hatch
+          fetch-depth: 0
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
@@ -43,6 +66,7 @@ jobs:
 
       - name: 📦 Build and Push Docker Image
         uses: docker/build-push-action@v2
+        if: (matrix.image == 'local' && github.event_name == 'workflow_dispatch') && github.event.inputs.khoj == 'true' || (matrix.image == 'local' && github.event_name == 'push')
         with:
           context: .
           file: Dockerfile
@@ -50,7 +74,22 @@ jobs:
           push: true
           tags: |
             ghcr.io/${{ github.repository }}:${{ env.DOCKER_IMAGE_TAG }}
-            ${{ github.ref_type == 'tag' && 'ghcr.io/${{ github.repository }}-cloud:latest' || '' }}
+            ${{ github.ref_type == 'tag' && format('ghcr.io/{0}:latest', github.repository) || '' }}
+          build-args: |
+            VERSION=${{ steps.hatch.outputs.version }}
+            PORT=42110
+
+      - name: 📦️⛅️ Build and Push Cloud Docker Image
+        uses: docker/build-push-action@v2
+        if: (matrix.image == 'cloud' && github.event_name == 'workflow_dispatch') && github.event.inputs.khoj-cloud == 'true' || (matrix.image == 'cloud' && github.event_name == 'push')
+        with:
+          context: .
+          file: prod.Dockerfile
+          platforms: linux/amd64
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}-cloud:${{ env.DOCKER_IMAGE_TAG }}
+            ${{ github.ref_type == 'tag' && format('ghcr.io/{0}-cloud:latest', github.repository) || '' }}
           build-args: |
             VERSION=${{ steps.hatch.outputs.version }}
             PORT=42110

.github/workflows/dockerize_production.yml

@@ -1,60 +0,0 @@
-name: dockerize production
-
-on:
-  pull_request:
-    paths:
-      - src/khoj/**
-      - pyproject.toml
-      - prod.Dockerfile
-      - .github/workflows/dockerize_production.yml
-  push:
-    tags:
-      - "*"
-    branches:
-      - master
-    paths:
-      - src/khoj/**
-      - pyproject.toml
-      - prod.Dockerfile
-      - .github/workflows/dockerize_production.yml
-  workflow_dispatch:
-
-env:
-  # Tag Image with tag name on release, else with 'pre' if push to master else with 'dev' if push to PR branch
-  DOCKER_IMAGE_TAG: ${{ github.ref_type == 'tag' && github.ref_name || github.event_name == 'pull_request' && 'dev' || 'pre' }}
-
-jobs:
-  build:
-    name: Build Production Docker Image, Push to Container Registry
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.PAT }}
-
-      - name: Get App Version
-        id: hatch
-        run: echo "::set-output name=version::$(pipx run hatch version)"
-
-      - name: 📦 Build and Push Docker Image
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          file: prod.Dockerfile
-          platforms: linux/amd64
-          push: true
-          tags: |
-            ghcr.io/${{ github.repository }}-cloud:${{ env.DOCKER_IMAGE_TAG }}
-            ${{ github.ref_type == 'tag' && 'ghcr.io/${{ github.repository }}-cloud:latest' || '' }}
-          build-args: |
-            VERSION=${{ steps.hatch.outputs.version }}
-            PORT=42110

config/khoj_docker.yml

@@ -1,51 +0,0 @@
-content-type:
-  # The /data/folder/ prefix to the folders is here because this is
-  # the directory to which the local files are copied in the docker-compose.
-  # If changing, the docker-compose volumes should also be changed to match.
-  org:
-    input-files: null
-    input-filter: ["/data/org/**/*.org"]
-    compressed-jsonl: "/data/embeddings/notes.jsonl.gz"
-    embeddings-file: "/data/embeddings/note_embeddings.pt"
-    index_heading_entries: false
-
-  markdown:
-    input-files: null
-    input-filter: ["/data/markdown/**/*.markdown"]
-    compressed-jsonl: "/data/embeddings/markdown.jsonl.gz"
-    embeddings-file: "/data/embeddings/markdown_embeddings.pt"
-
-  pdf:
-    input-files: null
-    input-filter: ["/data/pdf/**/*.pdf"]
-    compressed-jsonl: "/data/embeddings/pdf.jsonl.gz"
-    embeddings-file: "/data/embeddings/pdf_embeddings.pt"
-
-  image:
-    input-directories: ["/data/images/"]
-    embeddings-file: "/data/embeddings/image_embeddings.pt"
-    batch-size: 50
-    use-xmp-metadata: false
-
-  notion: null
-  github: null
-  plugins: null
-
-search-type:
-  symmetric: null
-  asymmetric:
-    encoder: "sentence-transformers/multi-qa-MiniLM-L6-cos-v1"
-    cross-encoder: "cross-encoder/ms-marco-MiniLM-L-6-v2"
-    model_directory: "/data/models/asymmetric"
-  image:
-    encoder: "sentence-transformers/clip-ViT-B-32"
-    model_directory: "/data/models/image_encoder"
-
-processor:
-  conversation:
-    conversation-logfile: "/data/embeddings/conversation_logs.json"
-    enable-offline-chat: false
-    openai: null
-
-app:
-  should_log_telemetry: true

config/khoj_sample.yml

@@ -1,57 +0,0 @@
-content-type:
-  org:
-    input-files:  # ["/path/to/org-file.org"]  REQUIRED IF input-filter IS NOT SET OR
-    input-filter: # ["/path/to/org/*.org"]     REQUIRED IF input-files IS NOT SET
-    compressed-jsonl: "~/.khoj/content/org/org.jsonl.gz"
-    embeddings-file: "~/.khoj/content/org/org_embeddings.pt"
-    index_heading_entries: false  # Set to true to index entries with empty body
-
-  markdown:
-    input-files:  # ["/path/to/markdown-file.md"]  REQUIRED IF input-filter IS NOT SET OR
-    input-filter: # ["/path/to/markdown/*.md"]     REQUIRED IF input-files IS NOT SET
-    compressed-jsonl: "~/.khoj/content/markdown/markdown.jsonl.gz"
-    embeddings-file: "~/.khoj/content/markdown/markdown_embeddings.pt"
-
-  ledger:
-    input-files:  # ["/path/to/ledger-file.beancount"]  REQUIRED IF input-filter is not set OR
-    input-filter: # ["/path/to/ledger/*.beancount"]     REQUIRED IF input-files is not set
-    compressed-jsonl: "~/.khoj/content/ledger/ledger.jsonl.gz"
-    embeddings-file: "~/.khoj/content/ledger/ledger_embeddings.pt"
-
-  image:
-    input-directories: # ["/path/to/images/"]       REQUIRED IF input-filter IS NOT SET OR
-    input-filter:      # ["/path/to/images/*.jpg"]  REQUIRED IF input-directories IS NOT SET
-    embeddings-file: "~/.khoj/content/image/image_embeddings.pt"
-    batch-size: 50
-    use-xmp-metadata: false
-
-  music:
-    input-files:  # ["/path/to/music-file.org"]  REQUIRED IF input-filter IS NOT SET OR
-    input-filter: # ["/path/to/music/*.org"]     REQUIRED IF input-files IS NOT SET
-    compressed-jsonl: "~/.khoj/content/music/music.jsonl.gz"
-    embeddings-file: "~/.khoj/content/music/music_embeddings.pt"
-
-search-type:
-  symmetric:
-    encoder: "sentence-transformers/all-MiniLM-L6-v2"
-    cross-encoder: "cross-encoder/ms-marco-MiniLM-L-6-v2"
-    encoder-type: sentence_transformers.SentenceTransformer
-    model_directory: "~/.khoj/search/symmetric/"
-
-  asymmetric:
-    encoder: "sentence-transformers/multi-qa-MiniLM-L6-cos-v1"
-    cross-encoder: "cross-encoder/ms-marco-MiniLM-L-6-v2"
-    encoder-type: sentence_transformers.SentenceTransformer
-    model_directory: "~/.khoj/search/asymmetric/"
-
-  image:
-    encoder: "sentence-transformers/clip-ViT-B-32"
-    encoder-type: sentence_transformers.SentenceTransformer
-    model_directory: "~/.khoj/search/image/"
-
-processor:
-  conversation:
-    openai-api-key: # "YOUR_OPENAI_API_KEY"
-    model: "text-davinci-003"
-    chat-model: "gpt-3.5-turbo"
-    conversation-logfile: "~/.khoj/processor/conversation/conversation_logs.json"