From 3806cce2e6499b56c4f74253f3a25edb1e9d61d2 Mon Sep 17 00:00:00 2001 From: sabaimran Date: Mon, 18 Nov 2024 16:05:12 -0800 Subject: [PATCH] TEMP commit just to show a proof of concept of how sqlite can be supported with the khoj backend - note that vector search / lookup is not yet working, but probably should be feasiable with an abstraction layer on top of the APIs which are doing searches. If vendor.sqlite, search this way, else search this way --- src/khoj/app/settings.py | 23 ++++--- src/khoj/database/adapters/__init__.py | 25 +++---- ...versation_temp_id_alter_conversation_id.py | 16 +++++ ..._agent_tools_agent_input_tools_and_more.py | 54 +++++++-------- .../0068_alter_agent_output_modes.py | 22 +++---- ...nt_input_tools_alter_agent_output_modes.py | 66 +++++++++---------- src/khoj/database/models/__init__.py | 8 +-- 7 files changed, 114 insertions(+), 100 deletions(-) diff --git a/src/khoj/app/settings.py b/src/khoj/app/settings.py index 020b2b2e..c448235f 100644 --- a/src/khoj/app/settings.py +++ b/src/khoj/app/settings.py @@ -115,16 +115,23 @@ CLOSE_CONNECTIONS_AFTER_REQUEST = True # Database # https://docs.djangoproject.com/en/4.2/ref/settings/#databases DATA_UPLOAD_MAX_NUMBER_FIELDS = 20000 +# DATABASES = { +# "default": { +# "ENGINE": "django.db.backends.postgresql", +# "HOST": os.getenv("POSTGRES_HOST", "localhost"), +# "PORT": os.getenv("POSTGRES_PORT", "5432"), +# "USER": os.getenv("POSTGRES_USER", "postgres"), +# "NAME": os.getenv("POSTGRES_DB", "khoj"), +# "PASSWORD": os.getenv("POSTGRES_PASSWORD", "postgres"), +# "CONN_MAX_AGE": 0, +# "CONN_HEALTH_CHECKS": True, +# } +# } + DATABASES = { "default": { - "ENGINE": "django.db.backends.postgresql", - "HOST": os.getenv("POSTGRES_HOST", "localhost"), - "PORT": os.getenv("POSTGRES_PORT", "5432"), - "USER": os.getenv("POSTGRES_USER", "postgres"), - "NAME": os.getenv("POSTGRES_DB", "khoj"), - "PASSWORD": os.getenv("POSTGRES_PASSWORD", "postgres"), - "CONN_MAX_AGE": 0, - "CONN_HEALTH_CHECKS": True, + "ENGINE": "django.db.backends.sqlite3", + "NAME": os.path.join(BASE_DIR, "db.sqlite3"), } } diff --git a/src/khoj/database/adapters/__init__.py b/src/khoj/database/adapters/__init__.py index fbfd5423..c8714f3f 100644 --- a/src/khoj/database/adapters/__init__.py +++ b/src/khoj/database/adapters/__init__.py @@ -689,11 +689,12 @@ class AgentAdapters: # TODO Update this to allow any public agent that's officially approved once that experience is launched public_query &= Q(managed_by_admin=True) if user: - return ( - Agent.objects.filter(public_query | Q(creator=user)) - .distinct() - .order_by("created_at") - .prefetch_related("creator", "chat_model", "fileobject_set") + return list( + set( + Agent.objects.filter(public_query | Q(creator=user)) + .order_by("created_at") + .prefetch_related("creator", "chat_model", "fileobject_set") + ) ) return ( Agent.objects.filter(public_query) @@ -1584,18 +1585,12 @@ class EntryAdapters: async def aget_agent_entry_filepaths(agent: Agent): if agent is None: return [] - return await sync_to_async(set)( - Entry.objects.filter(agent=agent).distinct("file_path").values_list("file_path", flat=True) - ) + return await sync_to_async(set)(Entry.objects.filter(agent=agent).values_list("file_path", flat=True)) @staticmethod @require_valid_user def get_all_filenames_by_source(user: KhojUser, file_source: str): - return ( - Entry.objects.filter(user=user, file_source=file_source) - .distinct("file_path") - .values_list("file_path", flat=True) - ) + return Entry.objects.filter(user=user, file_source=file_source).values_list("file_path", flat=True) @staticmethod @require_valid_user @@ -1698,12 +1693,12 @@ class EntryAdapters: @staticmethod @require_valid_user def get_unique_file_types(user: KhojUser): - return Entry.objects.filter(user=user).values_list("file_type", flat=True).distinct() + return list(set(Entry.objects.filter(user=user).values_list("file_type", flat=True))) @staticmethod @require_valid_user def get_unique_file_sources(user: KhojUser): - return Entry.objects.filter(user=user).values_list("file_source", flat=True).distinct().all() + return list(set(Entry.objects.filter(user=user).values_list("file_source", flat=True))) class AutomationAdapters: diff --git a/src/khoj/database/migrations/0064_remove_conversation_temp_id_alter_conversation_id.py b/src/khoj/database/migrations/0064_remove_conversation_temp_id_alter_conversation_id.py index 16d76d10..3806dab1 100644 --- a/src/khoj/database/migrations/0064_remove_conversation_temp_id_alter_conversation_id.py +++ b/src/khoj/database/migrations/0064_remove_conversation_temp_id_alter_conversation_id.py @@ -57,6 +57,22 @@ def enable_triggers(apps, schema_editor): schema_editor.execute('ALTER TABLE "database_conversation" ENABLE TRIGGER ALL;') +def rename_conversation_id_to_temp_id(apps, schema_editor): + # if sqlite + if schema_editor.connection.vendor == "sqlite": + pass + else: + schema_editor.execute('ALTER TABLE "database_conversation" RENAME COLUMN "id" TO "temp_id";') + + +def rename_temp_id_to_id(apps, schema_editor): + # if sqlite + if schema_editor.connection.vendor == "sqlite": + pass + else: + schema_editor.execute('ALTER TABLE "database_conversation" RENAME COLUMN "temp_id" TO "id";') + + class Migration(migrations.Migration): dependencies = [ ("database", "0063_conversation_temp_id"), diff --git a/src/khoj/database/migrations/0066_remove_agent_tools_agent_input_tools_and_more.py b/src/khoj/database/migrations/0066_remove_agent_tools_agent_input_tools_and_more.py index 167d5cc5..48db9a98 100644 --- a/src/khoj/database/migrations/0066_remove_agent_tools_agent_input_tools_and_more.py +++ b/src/khoj/database/migrations/0066_remove_agent_tools_agent_input_tools_and_more.py @@ -14,33 +14,33 @@ class Migration(migrations.Migration): model_name="agent", name="tools", ), - migrations.AddField( - model_name="agent", - name="input_tools", - field=django.contrib.postgres.fields.ArrayField( - base_field=models.CharField( - choices=[ - ("general", "General"), - ("online", "Online"), - ("notes", "Notes"), - ("summarize", "Summarize"), - ("webpage", "Webpage"), - ], - max_length=200, - ), - default=list, - size=None, - ), - ), - migrations.AddField( - model_name="agent", - name="output_modes", - field=django.contrib.postgres.fields.ArrayField( - base_field=models.CharField(choices=[("text", "Text"), ("image", "Image")], max_length=200), - default=list, - size=None, - ), - ), + # migrations.AddField( + # model_name="agent", + # name="input_tools", + # field=django.contrib.postgres.fields.ArrayField( + # base_field=models.CharField( + # choices=[ + # ("general", "General"), + # ("online", "Online"), + # ("notes", "Notes"), + # ("summarize", "Summarize"), + # ("webpage", "Webpage"), + # ], + # max_length=200, + # ), + # default=list, + # size=None, + # ), + # ), + # migrations.AddField( + # model_name="agent", + # name="output_modes", + # field=django.contrib.postgres.fields.ArrayField( + # base_field=models.CharField(choices=[("text", "Text"), ("image", "Image")], max_length=200), + # default=list, + # size=None, + # ), + # ), migrations.AlterField( model_name="agent", name="style_icon", diff --git a/src/khoj/database/migrations/0068_alter_agent_output_modes.py b/src/khoj/database/migrations/0068_alter_agent_output_modes.py index 31c01bd4..4a350984 100644 --- a/src/khoj/database/migrations/0068_alter_agent_output_modes.py +++ b/src/khoj/database/migrations/0068_alter_agent_output_modes.py @@ -10,15 +10,15 @@ class Migration(migrations.Migration): ] operations = [ - migrations.AlterField( - model_name="agent", - name="output_modes", - field=django.contrib.postgres.fields.ArrayField( - base_field=models.CharField( - choices=[("text", "Text"), ("image", "Image"), ("automation", "Automation")], max_length=200 - ), - default=list, - size=None, - ), - ), + # migrations.AlterField( + # model_name="agent", + # name="output_modes", + # field=django.contrib.postgres.fields.ArrayField( + # base_field=models.CharField( + # choices=[("text", "Text"), ("image", "Image"), ("automation", "Automation")], max_length=200 + # ), + # default=list, + # size=None, + # ), + # ), ] diff --git a/src/khoj/database/migrations/0070_alter_agent_input_tools_alter_agent_output_modes.py b/src/khoj/database/migrations/0070_alter_agent_input_tools_alter_agent_output_modes.py index 74dfc229..e9b8446f 100644 --- a/src/khoj/database/migrations/0070_alter_agent_input_tools_alter_agent_output_modes.py +++ b/src/khoj/database/migrations/0070_alter_agent_input_tools_alter_agent_output_modes.py @@ -10,37 +10,37 @@ class Migration(migrations.Migration): ] operations = [ - migrations.AlterField( - model_name="agent", - name="input_tools", - field=django.contrib.postgres.fields.ArrayField( - base_field=models.CharField( - choices=[ - ("general", "General"), - ("online", "Online"), - ("notes", "Notes"), - ("summarize", "Summarize"), - ("webpage", "Webpage"), - ], - max_length=200, - ), - blank=True, - default=list, - null=True, - size=None, - ), - ), - migrations.AlterField( - model_name="agent", - name="output_modes", - field=django.contrib.postgres.fields.ArrayField( - base_field=models.CharField( - choices=[("text", "Text"), ("image", "Image"), ("automation", "Automation")], max_length=200 - ), - blank=True, - default=list, - null=True, - size=None, - ), - ), + # migrations.AlterField( + # model_name="agent", + # name="input_tools", + # field=django.contrib.postgres.fields.ArrayField( + # base_field=models.CharField( + # choices=[ + # ("general", "General"), + # ("online", "Online"), + # ("notes", "Notes"), + # ("summarize", "Summarize"), + # ("webpage", "Webpage"), + # ], + # max_length=200, + # ), + # blank=True, + # default=list, + # null=True, + # size=None, + # ), + # ), + # migrations.AlterField( + # model_name="agent", + # name="output_modes", + # field=django.contrib.postgres.fields.ArrayField( + # base_field=models.CharField( + # choices=[("text", "Text"), ("image", "Image"), ("automation", "Automation")], max_length=200 + # ), + # blank=True, + # default=list, + # null=True, + # size=None, + # ), + # ), ] diff --git a/src/khoj/database/models/__init__.py b/src/khoj/database/models/__init__.py index 8f6c5c78..10d7ad2b 100644 --- a/src/khoj/database/models/__init__.py +++ b/src/khoj/database/models/__init__.py @@ -181,12 +181,8 @@ class Agent(BaseModel): ) # Creator will only be null when the agents are managed by admin name = models.CharField(max_length=200) personality = models.TextField() - input_tools = ArrayField( - models.CharField(max_length=200, choices=InputToolOptions.choices), default=list, null=True, blank=True - ) - output_modes = ArrayField( - models.CharField(max_length=200, choices=OutputModeOptions.choices), default=list, null=True, blank=True - ) + input_tools = models.JSONField(default=list, null=True) + output_modes = models.JSONField(default=list, null=True) managed_by_admin = models.BooleanField(default=False) chat_model = models.ForeignKey(ChatModelOptions, on_delete=models.CASCADE) slug = models.CharField(max_length=200, unique=True)