diff --git a/src/khoj/app/settings.py b/src/khoj/app/settings.py index 020b2b2e..c448235f 100644 --- a/src/khoj/app/settings.py +++ b/src/khoj/app/settings.py @@ -115,16 +115,23 @@ CLOSE_CONNECTIONS_AFTER_REQUEST = True # Database # https://docs.djangoproject.com/en/4.2/ref/settings/#databases DATA_UPLOAD_MAX_NUMBER_FIELDS = 20000 +# DATABASES = { +# "default": { +# "ENGINE": "django.db.backends.postgresql", +# "HOST": os.getenv("POSTGRES_HOST", "localhost"), +# "PORT": os.getenv("POSTGRES_PORT", "5432"), +# "USER": os.getenv("POSTGRES_USER", "postgres"), +# "NAME": os.getenv("POSTGRES_DB", "khoj"), +# "PASSWORD": os.getenv("POSTGRES_PASSWORD", "postgres"), +# "CONN_MAX_AGE": 0, +# "CONN_HEALTH_CHECKS": True, +# } +# } + DATABASES = { "default": { - "ENGINE": "django.db.backends.postgresql", - "HOST": os.getenv("POSTGRES_HOST", "localhost"), - "PORT": os.getenv("POSTGRES_PORT", "5432"), - "USER": os.getenv("POSTGRES_USER", "postgres"), - "NAME": os.getenv("POSTGRES_DB", "khoj"), - "PASSWORD": os.getenv("POSTGRES_PASSWORD", "postgres"), - "CONN_MAX_AGE": 0, - "CONN_HEALTH_CHECKS": True, + "ENGINE": "django.db.backends.sqlite3", + "NAME": os.path.join(BASE_DIR, "db.sqlite3"), } } diff --git a/src/khoj/database/adapters/__init__.py b/src/khoj/database/adapters/__init__.py index fbfd5423..c8714f3f 100644 --- a/src/khoj/database/adapters/__init__.py +++ b/src/khoj/database/adapters/__init__.py @@ -689,11 +689,12 @@ class AgentAdapters: # TODO Update this to allow any public agent that's officially approved once that experience is launched public_query &= Q(managed_by_admin=True) if user: - return ( - Agent.objects.filter(public_query | Q(creator=user)) - .distinct() - .order_by("created_at") - .prefetch_related("creator", "chat_model", "fileobject_set") + return list( + set( + Agent.objects.filter(public_query | Q(creator=user)) + .order_by("created_at") + .prefetch_related("creator", "chat_model", "fileobject_set") + ) ) return ( Agent.objects.filter(public_query) @@ -1584,18 +1585,12 @@ class EntryAdapters: async def aget_agent_entry_filepaths(agent: Agent): if agent is None: return [] - return await sync_to_async(set)( - Entry.objects.filter(agent=agent).distinct("file_path").values_list("file_path", flat=True) - ) + return await sync_to_async(set)(Entry.objects.filter(agent=agent).values_list("file_path", flat=True)) @staticmethod @require_valid_user def get_all_filenames_by_source(user: KhojUser, file_source: str): - return ( - Entry.objects.filter(user=user, file_source=file_source) - .distinct("file_path") - .values_list("file_path", flat=True) - ) + return Entry.objects.filter(user=user, file_source=file_source).values_list("file_path", flat=True) @staticmethod @require_valid_user @@ -1698,12 +1693,12 @@ class EntryAdapters: @staticmethod @require_valid_user def get_unique_file_types(user: KhojUser): - return Entry.objects.filter(user=user).values_list("file_type", flat=True).distinct() + return list(set(Entry.objects.filter(user=user).values_list("file_type", flat=True))) @staticmethod @require_valid_user def get_unique_file_sources(user: KhojUser): - return Entry.objects.filter(user=user).values_list("file_source", flat=True).distinct().all() + return list(set(Entry.objects.filter(user=user).values_list("file_source", flat=True))) class AutomationAdapters: diff --git a/src/khoj/database/migrations/0064_remove_conversation_temp_id_alter_conversation_id.py b/src/khoj/database/migrations/0064_remove_conversation_temp_id_alter_conversation_id.py index 16d76d10..3806dab1 100644 --- a/src/khoj/database/migrations/0064_remove_conversation_temp_id_alter_conversation_id.py +++ b/src/khoj/database/migrations/0064_remove_conversation_temp_id_alter_conversation_id.py @@ -57,6 +57,22 @@ def enable_triggers(apps, schema_editor): schema_editor.execute('ALTER TABLE "database_conversation" ENABLE TRIGGER ALL;') +def rename_conversation_id_to_temp_id(apps, schema_editor): + # if sqlite + if schema_editor.connection.vendor == "sqlite": + pass + else: + schema_editor.execute('ALTER TABLE "database_conversation" RENAME COLUMN "id" TO "temp_id";') + + +def rename_temp_id_to_id(apps, schema_editor): + # if sqlite + if schema_editor.connection.vendor == "sqlite": + pass + else: + schema_editor.execute('ALTER TABLE "database_conversation" RENAME COLUMN "temp_id" TO "id";') + + class Migration(migrations.Migration): dependencies = [ ("database", "0063_conversation_temp_id"), diff --git a/src/khoj/database/migrations/0066_remove_agent_tools_agent_input_tools_and_more.py b/src/khoj/database/migrations/0066_remove_agent_tools_agent_input_tools_and_more.py index 167d5cc5..48db9a98 100644 --- a/src/khoj/database/migrations/0066_remove_agent_tools_agent_input_tools_and_more.py +++ b/src/khoj/database/migrations/0066_remove_agent_tools_agent_input_tools_and_more.py @@ -14,33 +14,33 @@ class Migration(migrations.Migration): model_name="agent", name="tools", ), - migrations.AddField( - model_name="agent", - name="input_tools", - field=django.contrib.postgres.fields.ArrayField( - base_field=models.CharField( - choices=[ - ("general", "General"), - ("online", "Online"), - ("notes", "Notes"), - ("summarize", "Summarize"), - ("webpage", "Webpage"), - ], - max_length=200, - ), - default=list, - size=None, - ), - ), - migrations.AddField( - model_name="agent", - name="output_modes", - field=django.contrib.postgres.fields.ArrayField( - base_field=models.CharField(choices=[("text", "Text"), ("image", "Image")], max_length=200), - default=list, - size=None, - ), - ), + # migrations.AddField( + # model_name="agent", + # name="input_tools", + # field=django.contrib.postgres.fields.ArrayField( + # base_field=models.CharField( + # choices=[ + # ("general", "General"), + # ("online", "Online"), + # ("notes", "Notes"), + # ("summarize", "Summarize"), + # ("webpage", "Webpage"), + # ], + # max_length=200, + # ), + # default=list, + # size=None, + # ), + # ), + # migrations.AddField( + # model_name="agent", + # name="output_modes", + # field=django.contrib.postgres.fields.ArrayField( + # base_field=models.CharField(choices=[("text", "Text"), ("image", "Image")], max_length=200), + # default=list, + # size=None, + # ), + # ), migrations.AlterField( model_name="agent", name="style_icon", diff --git a/src/khoj/database/migrations/0068_alter_agent_output_modes.py b/src/khoj/database/migrations/0068_alter_agent_output_modes.py index 31c01bd4..4a350984 100644 --- a/src/khoj/database/migrations/0068_alter_agent_output_modes.py +++ b/src/khoj/database/migrations/0068_alter_agent_output_modes.py @@ -10,15 +10,15 @@ class Migration(migrations.Migration): ] operations = [ - migrations.AlterField( - model_name="agent", - name="output_modes", - field=django.contrib.postgres.fields.ArrayField( - base_field=models.CharField( - choices=[("text", "Text"), ("image", "Image"), ("automation", "Automation")], max_length=200 - ), - default=list, - size=None, - ), - ), + # migrations.AlterField( + # model_name="agent", + # name="output_modes", + # field=django.contrib.postgres.fields.ArrayField( + # base_field=models.CharField( + # choices=[("text", "Text"), ("image", "Image"), ("automation", "Automation")], max_length=200 + # ), + # default=list, + # size=None, + # ), + # ), ] diff --git a/src/khoj/database/migrations/0070_alter_agent_input_tools_alter_agent_output_modes.py b/src/khoj/database/migrations/0070_alter_agent_input_tools_alter_agent_output_modes.py index 74dfc229..e9b8446f 100644 --- a/src/khoj/database/migrations/0070_alter_agent_input_tools_alter_agent_output_modes.py +++ b/src/khoj/database/migrations/0070_alter_agent_input_tools_alter_agent_output_modes.py @@ -10,37 +10,37 @@ class Migration(migrations.Migration): ] operations = [ - migrations.AlterField( - model_name="agent", - name="input_tools", - field=django.contrib.postgres.fields.ArrayField( - base_field=models.CharField( - choices=[ - ("general", "General"), - ("online", "Online"), - ("notes", "Notes"), - ("summarize", "Summarize"), - ("webpage", "Webpage"), - ], - max_length=200, - ), - blank=True, - default=list, - null=True, - size=None, - ), - ), - migrations.AlterField( - model_name="agent", - name="output_modes", - field=django.contrib.postgres.fields.ArrayField( - base_field=models.CharField( - choices=[("text", "Text"), ("image", "Image"), ("automation", "Automation")], max_length=200 - ), - blank=True, - default=list, - null=True, - size=None, - ), - ), + # migrations.AlterField( + # model_name="agent", + # name="input_tools", + # field=django.contrib.postgres.fields.ArrayField( + # base_field=models.CharField( + # choices=[ + # ("general", "General"), + # ("online", "Online"), + # ("notes", "Notes"), + # ("summarize", "Summarize"), + # ("webpage", "Webpage"), + # ], + # max_length=200, + # ), + # blank=True, + # default=list, + # null=True, + # size=None, + # ), + # ), + # migrations.AlterField( + # model_name="agent", + # name="output_modes", + # field=django.contrib.postgres.fields.ArrayField( + # base_field=models.CharField( + # choices=[("text", "Text"), ("image", "Image"), ("automation", "Automation")], max_length=200 + # ), + # blank=True, + # default=list, + # null=True, + # size=None, + # ), + # ), ] diff --git a/src/khoj/database/models/__init__.py b/src/khoj/database/models/__init__.py index 8f6c5c78..10d7ad2b 100644 --- a/src/khoj/database/models/__init__.py +++ b/src/khoj/database/models/__init__.py @@ -181,12 +181,8 @@ class Agent(BaseModel): ) # Creator will only be null when the agents are managed by admin name = models.CharField(max_length=200) personality = models.TextField() - input_tools = ArrayField( - models.CharField(max_length=200, choices=InputToolOptions.choices), default=list, null=True, blank=True - ) - output_modes = ArrayField( - models.CharField(max_length=200, choices=OutputModeOptions.choices), default=list, null=True, blank=True - ) + input_tools = models.JSONField(default=list, null=True) + output_modes = models.JSONField(default=list, null=True) managed_by_admin = models.BooleanField(default=False) chat_model = models.ForeignKey(ChatModelOptions, on_delete=models.CASCADE) slug = models.CharField(max_length=200, unique=True)