From 79913d4c17b160cb0f7e3404bee78cb29031d369 Mon Sep 17 00:00:00 2001 From: sabaimran <65192171+sabaimran@users.noreply.github.com> Date: Thu, 28 Dec 2023 18:04:02 +0530 Subject: [PATCH] Add isort to the pre-commit configuration and apply it to the whole project (#595) * Apply isort to the entire repository * Fix missing import issues in text_to_entries * Fix imports in migration files --- .pre-commit-config.yaml | 7 ++++ src/khoj/app/settings.py | 2 +- src/khoj/app/urls.py | 2 +- src/khoj/configure.py | 35 ++++++++----------- src/khoj/database/adapters/__init__.py | 10 +++--- src/khoj/database/admin.py | 15 ++++---- src/khoj/database/migrations/0001_khojuser.py | 2 +- .../database/migrations/0002_googleuser.py | 2 +- .../migrations/0004_content_types_and_more.py | 7 ++-- .../migrations/0005_embeddings_corpus_id.py | 3 +- .../migrations/0006_embeddingsdates.py | 2 +- .../migrations/0007_add_conversation.py | 2 +- .../database/migrations/0009_khojapiuser.py | 2 +- .../0010_chatmodeloptions_and_more.py | 2 +- .../migrations/0011_merge_20231102_0138.py | 4 +-- .../database/migrations/0013_subscription.py | 2 +- .../0015_alter_subscription_user.py | 2 +- .../migrations/0020_reflectivequestion.py | 2 +- .../migrations/0023_usersearchmodelconfig.py | 2 +- .../migrations/0024_alter_entry_embeddings.py | 2 +- src/khoj/database/models/__init__.py | 2 +- src/khoj/main.py | 6 ++-- .../migrate_offline_chat_default_model.py | 2 +- .../migrations/migrate_offline_chat_schema.py | 2 +- src/khoj/migrations/migrate_offline_model.py | 3 +- src/khoj/migrations/migrate_server_pg.py | 7 ++-- .../content/github/github_to_entries.py | 4 --- .../content/markdown/markdown_to_entries.py | 3 -- .../content/notion/notion_to_entries.py | 4 --- .../content/org_mode/org_to_entries.py | 3 -- .../processor/content/org_mode/orgnode.py | 4 +-- .../processor/content/pdf/pdf_to_entries.py | 6 +--- .../content/plaintext/plaintext_to_entries.py | 3 -- src/khoj/processor/content/text_to_entries.py | 22 ++++++------ .../conversation/offline/chat_model.py | 11 +++--- .../processor/conversation/offline/whisper.py | 4 +-- src/khoj/processor/conversation/openai/gpt.py | 8 ++--- .../processor/conversation/openai/utils.py | 14 +++----- .../processor/conversation/openai/whisper.py | 2 -- src/khoj/processor/conversation/prompts.py | 2 -- src/khoj/processor/conversation/utils.py | 12 +++---- src/khoj/processor/embeddings.py | 2 +- src/khoj/processor/tools/online_search.py | 5 +-- src/khoj/routers/api.py | 28 +++++++++------ src/khoj/routers/api_beta.py | 2 -- src/khoj/routers/auth.py | 20 +++++------ src/khoj/routers/helpers.py | 16 +++++---- src/khoj/routers/indexer.py | 5 ++- src/khoj/routers/subscription.py | 8 ++--- src/khoj/routers/web_client.py | 29 +++++++-------- src/khoj/search_filter/base_filter.py | 1 - src/khoj/search_filter/date_filter.py | 12 +++---- src/khoj/search_filter/file_filter.py | 5 +-- src/khoj/search_filter/word_filter.py | 5 +-- src/khoj/search_type/image_search.py | 22 ++++++------ src/khoj/search_type/text_search.py | 20 +++++------ src/khoj/utils/cli.py | 22 ++++++------ src/khoj/utils/config.py | 11 +++--- src/khoj/utils/fs_syncer.py | 12 +++++-- src/khoj/utils/helpers.py | 23 ++++++------ src/khoj/utils/initialization.py | 9 ++--- src/khoj/utils/jsonl.py | 5 +-- src/khoj/utils/models.py | 3 -- src/khoj/utils/rawconfig.py | 6 ++-- src/khoj/utils/state.py | 11 +++--- src/khoj/utils/yaml.py | 6 +--- src/telemetry/telemetry.py | 12 +++---- tests/conftest.py | 15 +++++--- tests/helpers.py | 11 +++--- tests/test_cli.py | 1 - tests/test_client.py | 3 -- tests/test_conversation_utils.py | 5 +-- tests/test_date_filter.py | 3 -- tests/test_gpt4all_chat_actors.py | 11 +++--- tests/test_gpt4all_chat_director.py | 5 +-- tests/test_helpers.py | 9 ++--- tests/test_image_search.py | 12 +++---- tests/test_markdown_to_entries.py | 4 +-- tests/test_multiple_users.py | 3 -- tests/test_openai_chat_actors.py | 6 +--- tests/test_openai_chat_director.py | 7 ++-- tests/test_org_to_entries.py | 7 ++-- tests/test_orgnode.py | 2 -- tests/test_pdf_to_entries.py | 3 -- tests/test_plaintext_to_entries.py | 3 -- tests/test_rawconfig.py | 4 +-- tests/test_text_search.py | 3 -- 87 files changed, 274 insertions(+), 374 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 77d4ec6c..444e182d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,6 +15,13 @@ repos: - id: check-toml - id: check-yaml +- repo: https://github.com/pycqa/isort + rev: 5.12.0 + hooks: + - id: isort + name: isort (python) + args: ["--profile", "black", "--filter-files"] + - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.0.0 hooks: diff --git a/src/khoj/app/settings.py b/src/khoj/app/settings.py index 86db4b12..8f7d409d 100644 --- a/src/khoj/app/settings.py +++ b/src/khoj/app/settings.py @@ -10,8 +10,8 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/4.2/ref/settings/ """ -from pathlib import Path import os +from pathlib import Path # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent diff --git a/src/khoj/app/urls.py b/src/khoj/app/urls.py index 39b4b1ef..2754270c 100644 --- a/src/khoj/app/urls.py +++ b/src/khoj/app/urls.py @@ -15,8 +15,8 @@ Including another URLconf 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ from django.contrib import admin -from django.urls import path from django.contrib.staticfiles.urls import staticfiles_urlpatterns +from django.urls import path urlpatterns = [ path("admin/", admin.site.urls), diff --git a/src/khoj/configure.py b/src/khoj/configure.py index 613884ed..0f9e5fef 100644 --- a/src/khoj/configure.py +++ b/src/khoj/configure.py @@ -1,44 +1,37 @@ -# Standard Packages -import logging import json +import logging +import os from enum import Enum from typing import Optional -import requests -import os -# External Packages import openai +import requests import schedule -from starlette.middleware.sessions import SessionMiddleware -from starlette.middleware.authentication import AuthenticationMiddleware -from starlette.requests import HTTPConnection - from starlette.authentication import ( AuthCredentials, AuthenticationBackend, SimpleUser, UnauthenticatedUser, ) +from starlette.middleware.authentication import AuthenticationMiddleware +from starlette.middleware.sessions import SessionMiddleware +from starlette.requests import HTTPConnection -# Internal Packages -from khoj.database.models import KhojUser, Subscription from khoj.database.adapters import ( ConversationAdapters, + SubscriptionState, + aget_user_subscription_state, get_all_users, get_or_create_search_models, - aget_user_subscription_state, - SubscriptionState, ) +from khoj.database.models import KhojUser, Subscription from khoj.processor.embeddings import CrossEncoderModel, EmbeddingsModel -from khoj.routers.indexer import configure_content, load_content, configure_search +from khoj.routers.indexer import configure_content, configure_search, load_content from khoj.utils import constants, state -from khoj.utils.config import ( - SearchType, -) +from khoj.utils.config import SearchType from khoj.utils.fs_syncer import collect_files from khoj.utils.rawconfig import FullConfig - logger = logging.getLogger(__name__) @@ -52,7 +45,7 @@ class UserAuthenticationBackend(AuthenticationBackend): def __init__( self, ): - from khoj.database.models import KhojUser, KhojApiUser + from khoj.database.models import KhojApiUser, KhojUser self.khojuser_manager = KhojUser.objects self.khojapiuser_manager = KhojApiUser.objects @@ -190,10 +183,10 @@ def configure_routes(app): # Import APIs here to setup search types before while configuring server from khoj.routers.api import api from khoj.routers.api_beta import api_beta - from khoj.routers.web_client import web_client - from khoj.routers.indexer import indexer from khoj.routers.auth import auth_router + from khoj.routers.indexer import indexer from khoj.routers.subscription import subscription_router + from khoj.routers.web_client import web_client app.include_router(api, prefix="/api") app.include_router(api_beta, prefix="/api/beta") diff --git a/src/khoj/database/adapters/__init__.py b/src/khoj/database/adapters/__init__.py index 2992ef37..159d9306 100644 --- a/src/khoj/database/adapters/__init__.py +++ b/src/khoj/database/adapters/__init__.py @@ -2,9 +2,9 @@ import math import random import secrets import sys -from datetime import date, datetime, timezone, timedelta -from typing import List, Optional, Type +from datetime import date, datetime, timedelta, timezone from enum import Enum +from typing import List, Optional, Type from asgiref.sync import sync_to_async from django.contrib.sessions.backends.db import SessionStore @@ -22,20 +22,18 @@ from khoj.database.models import ( GithubConfig, GithubRepoConfig, GoogleUser, - TextToImageModelConfig, KhojApiUser, KhojUser, NotionConfig, OfflineChatProcessorConversationConfig, OpenAIProcessorConversationConfig, + ReflectiveQuestion, SearchModelConfig, SpeechToTextModelOptions, Subscription, + TextToImageModelConfig, UserConversationConfig, UserSearchModelConfig, - OpenAIProcessorConversationConfig, - OfflineChatProcessorConversationConfig, - ReflectiveQuestion, ) from khoj.search_filter.date_filter import DateFilter from khoj.search_filter.file_filter import FileFilter diff --git a/src/khoj/database/admin.py b/src/khoj/database/admin.py index 5b5f1006..f8d084d8 100644 --- a/src/khoj/database/admin.py +++ b/src/khoj/database/admin.py @@ -5,22 +5,23 @@ from django.contrib import admin from django.contrib.auth.admin import UserAdmin from django.http import HttpResponse -# Register your models here. - from khoj.database.models import ( - KhojUser, ChatModelOptions, - OpenAIProcessorConversationConfig, + Conversation, + KhojUser, OfflineChatProcessorConversationConfig, + OpenAIProcessorConversationConfig, + ReflectiveQuestion, SearchModelConfig, SpeechToTextModelOptions, Subscription, - ReflectiveQuestion, - UserSearchModelConfig, TextToImageModelConfig, - Conversation, + UserSearchModelConfig, ) +# Register your models here. + + admin.site.register(KhojUser, UserAdmin) admin.site.register(ChatModelOptions) diff --git a/src/khoj/database/migrations/0001_khojuser.py b/src/khoj/database/migrations/0001_khojuser.py index f1420575..b7c321cc 100644 --- a/src/khoj/database/migrations/0001_khojuser.py +++ b/src/khoj/database/migrations/0001_khojuser.py @@ -2,8 +2,8 @@ import django.contrib.auth.models import django.contrib.auth.validators -from django.db import migrations, models import django.utils.timezone +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/src/khoj/database/migrations/0002_googleuser.py b/src/khoj/database/migrations/0002_googleuser.py index 478770d6..0a25d2fd 100644 --- a/src/khoj/database/migrations/0002_googleuser.py +++ b/src/khoj/database/migrations/0002_googleuser.py @@ -1,8 +1,8 @@ # Generated by Django 4.2.4 on 2023-09-18 23:24 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/src/khoj/database/migrations/0004_content_types_and_more.py b/src/khoj/database/migrations/0004_content_types_and_more.py index ec704e1f..f2e14ccb 100644 --- a/src/khoj/database/migrations/0004_content_types_and_more.py +++ b/src/khoj/database/migrations/0004_content_types_and_more.py @@ -1,10 +1,11 @@ # Generated by Django 4.2.5 on 2023-10-11 22:24 -from django.conf import settings -from django.db import migrations, models +import uuid + import django.db.models.deletion import pgvector.django -import uuid +from django.conf import settings +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/src/khoj/database/migrations/0005_embeddings_corpus_id.py b/src/khoj/database/migrations/0005_embeddings_corpus_id.py index 984953d6..83d41440 100644 --- a/src/khoj/database/migrations/0005_embeddings_corpus_id.py +++ b/src/khoj/database/migrations/0005_embeddings_corpus_id.py @@ -1,8 +1,9 @@ # Generated by Django 4.2.5 on 2023-10-13 02:39 -from django.db import migrations, models import uuid +from django.db import migrations, models + class Migration(migrations.Migration): dependencies = [ diff --git a/src/khoj/database/migrations/0006_embeddingsdates.py b/src/khoj/database/migrations/0006_embeddingsdates.py index 9d988ed8..8a78bfc6 100644 --- a/src/khoj/database/migrations/0006_embeddingsdates.py +++ b/src/khoj/database/migrations/0006_embeddingsdates.py @@ -1,7 +1,7 @@ # Generated by Django 4.2.5 on 2023-10-13 19:28 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/src/khoj/database/migrations/0007_add_conversation.py b/src/khoj/database/migrations/0007_add_conversation.py index 167b6cab..64ec24e4 100644 --- a/src/khoj/database/migrations/0007_add_conversation.py +++ b/src/khoj/database/migrations/0007_add_conversation.py @@ -1,8 +1,8 @@ # Generated by Django 4.2.5 on 2023-10-18 05:31 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/src/khoj/database/migrations/0009_khojapiuser.py b/src/khoj/database/migrations/0009_khojapiuser.py index 86b09ab3..f6e62cba 100644 --- a/src/khoj/database/migrations/0009_khojapiuser.py +++ b/src/khoj/database/migrations/0009_khojapiuser.py @@ -1,8 +1,8 @@ # Generated by Django 4.2.5 on 2023-10-26 17:02 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/src/khoj/database/migrations/0010_chatmodeloptions_and_more.py b/src/khoj/database/migrations/0010_chatmodeloptions_and_more.py index 9f3a491a..e6991ea9 100644 --- a/src/khoj/database/migrations/0010_chatmodeloptions_and_more.py +++ b/src/khoj/database/migrations/0010_chatmodeloptions_and_more.py @@ -1,8 +1,8 @@ # Generated by Django 4.2.4 on 2023-11-01 17:41 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/src/khoj/database/migrations/0011_merge_20231102_0138.py b/src/khoj/database/migrations/0011_merge_20231102_0138.py index 0206036e..bf6560c1 100644 --- a/src/khoj/database/migrations/0011_merge_20231102_0138.py +++ b/src/khoj/database/migrations/0011_merge_20231102_0138.py @@ -1,8 +1,8 @@ # Generated by Django 4.2.5 on 2023-11-02 01:38 -from django.db import migrations +from typing import Any, List -from typing import List, Any +from django.db import migrations class Migration(migrations.Migration): diff --git a/src/khoj/database/migrations/0013_subscription.py b/src/khoj/database/migrations/0013_subscription.py index 931cea12..63a90eb4 100644 --- a/src/khoj/database/migrations/0013_subscription.py +++ b/src/khoj/database/migrations/0013_subscription.py @@ -1,8 +1,8 @@ # Generated by Django 4.2.5 on 2023-11-09 01:27 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/src/khoj/database/migrations/0015_alter_subscription_user.py b/src/khoj/database/migrations/0015_alter_subscription_user.py index e4ba6ab0..5fdbf94c 100644 --- a/src/khoj/database/migrations/0015_alter_subscription_user.py +++ b/src/khoj/database/migrations/0015_alter_subscription_user.py @@ -1,8 +1,8 @@ # Generated by Django 4.2.5 on 2023-11-11 05:39 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/src/khoj/database/migrations/0020_reflectivequestion.py b/src/khoj/database/migrations/0020_reflectivequestion.py index aefb73bf..1ad60cd2 100644 --- a/src/khoj/database/migrations/0020_reflectivequestion.py +++ b/src/khoj/database/migrations/0020_reflectivequestion.py @@ -1,8 +1,8 @@ # Generated by Django 4.2.7 on 2023-11-20 01:13 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/src/khoj/database/migrations/0023_usersearchmodelconfig.py b/src/khoj/database/migrations/0023_usersearchmodelconfig.py index 0aec99f0..bd65f528 100644 --- a/src/khoj/database/migrations/0023_usersearchmodelconfig.py +++ b/src/khoj/database/migrations/0023_usersearchmodelconfig.py @@ -1,8 +1,8 @@ # Generated by Django 4.2.7 on 2023-12-19 15:44 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/src/khoj/database/migrations/0024_alter_entry_embeddings.py b/src/khoj/database/migrations/0024_alter_entry_embeddings.py index a1bbf45d..425ff915 100644 --- a/src/khoj/database/migrations/0024_alter_entry_embeddings.py +++ b/src/khoj/database/migrations/0024_alter_entry_embeddings.py @@ -1,7 +1,7 @@ # Generated by Django 4.2.7 on 2023-12-20 07:27 -from django.db import migrations import pgvector.django +from django.db import migrations class Migration(migrations.Migration): diff --git a/src/khoj/database/models/__init__.py b/src/khoj/database/models/__init__.py index a9fa38f7..873e9628 100644 --- a/src/khoj/database/models/__init__.py +++ b/src/khoj/database/models/__init__.py @@ -1,7 +1,7 @@ import uuid -from django.db import models from django.contrib.auth.models import AbstractUser +from django.db import models from pgvector.django import VectorField diff --git a/src/khoj/main.py b/src/khoj/main.py index c99ef1a1..e06f0ff4 100644 --- a/src/khoj/main.py +++ b/src/khoj/main.py @@ -2,7 +2,7 @@ isort:skip_file """ -# Standard Packages + from contextlib import redirect_stdout import io import os @@ -18,7 +18,7 @@ from importlib.metadata import version warnings.filterwarnings("ignore", message=r"snapshot_download.py has been made private", category=FutureWarning) warnings.filterwarnings("ignore", message=r"legacy way to download files from the HF hub,", category=FutureWarning) -# External Packages + import uvicorn import django from fastapi import FastAPI @@ -69,7 +69,7 @@ app.add_middleware( # Set Locale locale.setlocale(locale.LC_ALL, "") -# Internal Packages. We do this after setting up Django so that Django features are accessible to the app. +# We import these packages after setting up Django so that Django features are accessible to the app. from khoj.configure import configure_routes, initialize_server, configure_middleware from khoj.utils import state from khoj.utils.cli import cli diff --git a/src/khoj/migrations/migrate_offline_chat_default_model.py b/src/khoj/migrations/migrate_offline_chat_default_model.py index b7f4cf94..831f2d9d 100644 --- a/src/khoj/migrations/migrate_offline_chat_default_model.py +++ b/src/khoj/migrations/migrate_offline_chat_default_model.py @@ -30,11 +30,11 @@ search-type: ... """ import logging + from packaging import version from khoj.utils.yaml import load_config_from_file, save_config_to_file - logger = logging.getLogger(__name__) diff --git a/src/khoj/migrations/migrate_offline_chat_schema.py b/src/khoj/migrations/migrate_offline_chat_schema.py index 873783a3..0c221652 100644 --- a/src/khoj/migrations/migrate_offline_chat_schema.py +++ b/src/khoj/migrations/migrate_offline_chat_schema.py @@ -34,11 +34,11 @@ search-type: ... """ import logging + from packaging import version from khoj.utils.yaml import load_config_from_file, save_config_to_file - logger = logging.getLogger(__name__) diff --git a/src/khoj/migrations/migrate_offline_model.py b/src/khoj/migrations/migrate_offline_model.py index 853ceb4b..6294a4e8 100644 --- a/src/khoj/migrations/migrate_offline_model.py +++ b/src/khoj/migrations/migrate_offline_model.py @@ -1,5 +1,6 @@ -import os import logging +import os + from packaging import version from khoj.utils.yaml import load_config_from_file, save_config_to_file diff --git a/src/khoj/migrations/migrate_server_pg.py b/src/khoj/migrations/migrate_server_pg.py index 097ae0d6..fbb26a1e 100644 --- a/src/khoj/migrations/migrate_server_pg.py +++ b/src/khoj/migrations/migrate_server_pg.py @@ -57,15 +57,16 @@ version: 0.15.0 """ import logging + from packaging import version -from khoj.utils.yaml import load_config_from_file, save_config_to_file from khoj.database.models import ( - OpenAIProcessorConversationConfig, - OfflineChatProcessorConversationConfig, ChatModelOptions, + OfflineChatProcessorConversationConfig, + OpenAIProcessorConversationConfig, SearchModelConfig, ) +from khoj.utils.yaml import load_config_from_file, save_config_to_file logger = logging.getLogger(__name__) diff --git a/src/khoj/processor/content/github/github_to_entries.py b/src/khoj/processor/content/github/github_to_entries.py index 1149227e..fb22e7a2 100644 --- a/src/khoj/processor/content/github/github_to_entries.py +++ b/src/khoj/processor/content/github/github_to_entries.py @@ -1,10 +1,8 @@ -# Standard Packages import logging import time from datetime import datetime from typing import Dict, List, Tuple, Union -# External Packages import requests from khoj.database.models import Entry as DbEntry @@ -12,8 +10,6 @@ from khoj.database.models import GithubConfig, KhojUser from khoj.processor.content.markdown.markdown_to_entries import MarkdownToEntries from khoj.processor.content.org_mode.org_to_entries import OrgToEntries from khoj.processor.content.text_to_entries import TextToEntries - -# Internal Packages from khoj.utils.helpers import timer from khoj.utils.rawconfig import Entry, GithubContentConfig, GithubRepoConfig diff --git a/src/khoj/processor/content/markdown/markdown_to_entries.py b/src/khoj/processor/content/markdown/markdown_to_entries.py index 2a4b6348..4cce4876 100644 --- a/src/khoj/processor/content/markdown/markdown_to_entries.py +++ b/src/khoj/processor/content/markdown/markdown_to_entries.py @@ -1,4 +1,3 @@ -# Standard Packages import logging import re from pathlib import Path @@ -8,8 +7,6 @@ import urllib3 from khoj.database.models import Entry as DbEntry from khoj.database.models import KhojUser - -# Internal Packages from khoj.processor.content.text_to_entries import TextToEntries from khoj.utils.constants import empty_escape_sequences from khoj.utils.helpers import timer diff --git a/src/khoj/processor/content/notion/notion_to_entries.py b/src/khoj/processor/content/notion/notion_to_entries.py index 21c7232d..bf9fa0fd 100644 --- a/src/khoj/processor/content/notion/notion_to_entries.py +++ b/src/khoj/processor/content/notion/notion_to_entries.py @@ -1,16 +1,12 @@ -# Standard Packages import logging from enum import Enum from typing import Tuple -# External Packages import requests from khoj.database.models import Entry as DbEntry from khoj.database.models import KhojUser, NotionConfig from khoj.processor.content.text_to_entries import TextToEntries - -# Internal Packages from khoj.utils.helpers import timer from khoj.utils.rawconfig import Entry, NotionContentConfig diff --git a/src/khoj/processor/content/org_mode/org_to_entries.py b/src/khoj/processor/content/org_mode/org_to_entries.py index c3f345f0..0e115f78 100644 --- a/src/khoj/processor/content/org_mode/org_to_entries.py +++ b/src/khoj/processor/content/org_mode/org_to_entries.py @@ -1,12 +1,9 @@ -# Standard Packages import logging from pathlib import Path from typing import Iterable, List, Tuple from khoj.database.models import Entry as DbEntry from khoj.database.models import KhojUser - -# Internal Packages from khoj.processor.content.org_mode import orgnode from khoj.processor.content.text_to_entries import TextToEntries from khoj.utils import state diff --git a/src/khoj/processor/content/org_mode/orgnode.py b/src/khoj/processor/content/org_mode/orgnode.py index 28f55c17..0c9bcb6e 100644 --- a/src/khoj/processor/content/org_mode/orgnode.py +++ b/src/khoj/processor/content/org_mode/orgnode.py @@ -33,10 +33,10 @@ headline and associated text from an org-mode file, and routines for constructing data structures of these classes. """ -import re import datetime -from pathlib import Path +import re from os.path import relpath +from pathlib import Path from typing import List indent_regex = re.compile(r"^ *") diff --git a/src/khoj/processor/content/pdf/pdf_to_entries.py b/src/khoj/processor/content/pdf/pdf_to_entries.py index 101e582b..efe57a21 100644 --- a/src/khoj/processor/content/pdf/pdf_to_entries.py +++ b/src/khoj/processor/content/pdf/pdf_to_entries.py @@ -1,17 +1,13 @@ -# Standard Packages import base64 import logging import os -from typing import List, Tuple from datetime import datetime +from typing import List, Tuple -# External Packages from langchain.document_loaders import PyMuPDFLoader from khoj.database.models import Entry as DbEntry from khoj.database.models import KhojUser - -# Internal Packages from khoj.processor.content.text_to_entries import TextToEntries from khoj.utils.helpers import timer from khoj.utils.rawconfig import Entry diff --git a/src/khoj/processor/content/plaintext/plaintext_to_entries.py b/src/khoj/processor/content/plaintext/plaintext_to_entries.py index cae88837..9ea8c654 100644 --- a/src/khoj/processor/content/plaintext/plaintext_to_entries.py +++ b/src/khoj/processor/content/plaintext/plaintext_to_entries.py @@ -1,4 +1,3 @@ -# Standard Packages import logging from pathlib import Path from typing import List, Tuple @@ -7,8 +6,6 @@ from bs4 import BeautifulSoup from khoj.database.models import Entry as DbEntry from khoj.database.models import KhojUser - -# Internal Packages from khoj.processor.content.text_to_entries import TextToEntries from khoj.utils.helpers import timer from khoj.utils.rawconfig import Entry diff --git a/src/khoj/processor/content/text_to_entries.py b/src/khoj/processor/content/text_to_entries.py index 0c092ed4..ea62cde3 100644 --- a/src/khoj/processor/content/text_to_entries.py +++ b/src/khoj/processor/content/text_to_entries.py @@ -1,21 +1,19 @@ -# Standard Packages -from abc import ABC, abstractmethod import hashlib -from itertools import repeat import logging import uuid +from abc import ABC, abstractmethod +from itertools import repeat +from typing import Any, Callable, List, Set, Tuple + from tqdm import tqdm -from typing import Callable, List, Tuple, Set, Any -from khoj.utils import state -from khoj.utils.helpers import is_none_or_empty, timer, batcher - -# Internal Packages -from khoj.utils.rawconfig import Entry -from khoj.search_filter.date_filter import DateFilter -from khoj.database.models import KhojUser, Entry as DbEntry, EntryDates from khoj.database.adapters import EntryAdapters, get_user_search_model_or_default - +from khoj.database.models import Entry as DbEntry +from khoj.database.models import EntryDates, KhojUser +from khoj.search_filter.date_filter import DateFilter +from khoj.utils import state +from khoj.utils.helpers import batcher, is_none_or_empty, timer +from khoj.utils.rawconfig import Entry logger = logging.getLogger(__name__) diff --git a/src/khoj/processor/conversation/offline/chat_model.py b/src/khoj/processor/conversation/offline/chat_model.py index 211e3cac..23a77bb2 100644 --- a/src/khoj/processor/conversation/offline/chat_model.py +++ b/src/khoj/processor/conversation/offline/chat_model.py @@ -1,14 +1,17 @@ -from typing import Iterator, Union, List, Any -from datetime import datetime import logging +from datetime import datetime from threading import Thread +from typing import Any, Iterator, List, Union from langchain.schema import ChatMessage -from khoj.processor.conversation.utils import ThreadedGenerator, generate_chatml_messages_with_context from khoj.processor.conversation import prompts -from khoj.utils.constants import empty_escape_sequences +from khoj.processor.conversation.utils import ( + ThreadedGenerator, + generate_chatml_messages_with_context, +) from khoj.utils import state +from khoj.utils.constants import empty_escape_sequences from khoj.utils.helpers import ConversationCommand, is_none_or_empty logger = logging.getLogger(__name__) diff --git a/src/khoj/processor/conversation/offline/whisper.py b/src/khoj/processor/conversation/offline/whisper.py index 56d2aaf5..d8dd4457 100644 --- a/src/khoj/processor/conversation/offline/whisper.py +++ b/src/khoj/processor/conversation/offline/whisper.py @@ -1,8 +1,6 @@ -# External Packages -from asgiref.sync import sync_to_async import whisper +from asgiref.sync import sync_to_async -# Internal Packages from khoj.utils import state diff --git a/src/khoj/processor/conversation/openai/gpt.py b/src/khoj/processor/conversation/openai/gpt.py index 7bebc26c..3038aa00 100644 --- a/src/khoj/processor/conversation/openai/gpt.py +++ b/src/khoj/processor/conversation/openai/gpt.py @@ -1,23 +1,19 @@ -# Standard Packages -import logging import json +import logging from datetime import datetime, timedelta from typing import Optional -# External Packages from langchain.schema import ChatMessage -# Internal Packages -from khoj.utils.constants import empty_escape_sequences from khoj.processor.conversation import prompts from khoj.processor.conversation.openai.utils import ( chat_completion_with_backoff, completion_with_backoff, ) from khoj.processor.conversation.utils import generate_chatml_messages_with_context +from khoj.utils.constants import empty_escape_sequences from khoj.utils.helpers import ConversationCommand, is_none_or_empty - logger = logging.getLogger(__name__) diff --git a/src/khoj/processor/conversation/openai/utils.py b/src/khoj/processor/conversation/openai/utils.py index a5868cb8..58b3f583 100644 --- a/src/khoj/processor/conversation/openai/utils.py +++ b/src/khoj/processor/conversation/openai/utils.py @@ -1,14 +1,12 @@ -# Standard Packages -import os import logging -from typing import Any +import os from threading import Thread +from typing import Any -# External Packages -from langchain.chat_models import ChatOpenAI -from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler -from langchain.callbacks.base import BaseCallbackManager import openai +from langchain.callbacks.base import BaseCallbackManager +from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler +from langchain.chat_models import ChatOpenAI from tenacity import ( before_sleep_log, retry, @@ -18,10 +16,8 @@ from tenacity import ( wait_random_exponential, ) -# Internal Packages from khoj.processor.conversation.utils import ThreadedGenerator - logger = logging.getLogger(__name__) diff --git a/src/khoj/processor/conversation/openai/whisper.py b/src/khoj/processor/conversation/openai/whisper.py index bd0e66df..dcb84f89 100644 --- a/src/khoj/processor/conversation/openai/whisper.py +++ b/src/khoj/processor/conversation/openai/whisper.py @@ -1,7 +1,5 @@ -# Standard Packages from io import BufferedReader -# External Packages from asgiref.sync import sync_to_async from openai import OpenAI diff --git a/src/khoj/processor/conversation/prompts.py b/src/khoj/processor/conversation/prompts.py index d0c4aba4..aef4776a 100644 --- a/src/khoj/processor/conversation/prompts.py +++ b/src/khoj/processor/conversation/prompts.py @@ -1,7 +1,5 @@ -# External Packages from langchain.prompts import PromptTemplate - ## Personality ## -- personality = PromptTemplate.from_template( diff --git a/src/khoj/processor/conversation/utils.py b/src/khoj/processor/conversation/utils.py index 4606efd9..32049918 100644 --- a/src/khoj/processor/conversation/utils.py +++ b/src/khoj/processor/conversation/utils.py @@ -1,22 +1,18 @@ -# Standard Packages -import logging -from time import perf_counter import json -from datetime import datetime +import logging import queue +from datetime import datetime +from time import perf_counter from typing import Any, Dict, List -import tiktoken -# External packages +import tiktoken from langchain.schema import ChatMessage from transformers import AutoTokenizer -# Internal Packages from khoj.database.adapters import ConversationAdapters from khoj.database.models import KhojUser from khoj.utils.helpers import merge_dicts - logger = logging.getLogger(__name__) model_to_prompt_size = { "gpt-3.5-turbo": 4096, diff --git a/src/khoj/processor/embeddings.py b/src/khoj/processor/embeddings.py index 392d402f..4cb01823 100644 --- a/src/khoj/processor/embeddings.py +++ b/src/khoj/processor/embeddings.py @@ -1,6 +1,6 @@ from typing import List -from sentence_transformers import SentenceTransformer, CrossEncoder +from sentence_transformers import CrossEncoder, SentenceTransformer from torch import nn from khoj.utils.helpers import get_device diff --git a/src/khoj/processor/tools/online_search.py b/src/khoj/processor/tools/online_search.py index f014fb4a..f79ad8a6 100644 --- a/src/khoj/processor/tools/online_search.py +++ b/src/khoj/processor/tools/online_search.py @@ -1,7 +1,8 @@ -import requests import json -import os import logging +import os + +import requests from khoj.routers.helpers import generate_online_subqueries diff --git a/src/khoj/routers/api.py b/src/khoj/routers/api.py index c16a81e5..7b4ed311 100644 --- a/src/khoj/routers/api.py +++ b/src/khoj/routers/api.py @@ -1,25 +1,26 @@ -# Standard Packages import concurrent.futures import json import logging import math import os import time -from typing import Any, Dict, List, Optional, Union import uuid +from typing import Any, Dict, List, Optional, Union -# External Packages -from fastapi import APIRouter, Depends, File, HTTPException, Request, UploadFile from asgiref.sync import sync_to_async +from fastapi import APIRouter, Depends, File, HTTPException, Request, UploadFile from fastapi.requests import Request from fastapi.responses import Response, StreamingResponse from starlette.authentication import requires -# Internal Packages from khoj.configure import configure_server from khoj.database import adapters -from khoj.database.adapters import ConversationAdapters, EntryAdapters, get_user_search_model_or_default -from khoj.database.models import ChatModelOptions, SpeechToTextModelOptions +from khoj.database.adapters import ( + ConversationAdapters, + EntryAdapters, + get_user_search_model_or_default, +) +from khoj.database.models import ChatModelOptions from khoj.database.models import Entry as DbEntry from khoj.database.models import ( GithubConfig, @@ -29,6 +30,7 @@ from khoj.database.models import ( LocalPdfConfig, LocalPlaintextConfig, NotionConfig, + SpeechToTextModelOptions, ) from khoj.processor.conversation.offline.chat_model import extract_questions_offline from khoj.processor.conversation.offline.whisper import transcribe_audio_offline @@ -40,13 +42,13 @@ from khoj.processor.tools.online_search import search_with_google from khoj.routers.helpers import ( ApiUserRateLimiter, CommonQueryParams, + ConversationCommandRateLimiter, agenerate_chat_response, get_conversation_command, - text_to_image, is_ready_to_chat, + text_to_image, update_telemetry_state, validate_conversation_config, - ConversationCommandRateLimiter, ) from khoj.search_filter.date_filter import DateFilter from khoj.search_filter.file_filter import FileFilter @@ -62,7 +64,13 @@ from khoj.utils.helpers import ( is_none_or_empty, timer, ) -from khoj.utils.rawconfig import FullConfig, GithubContentConfig, NotionContentConfig, SearchConfig, SearchResponse +from khoj.utils.rawconfig import ( + FullConfig, + GithubContentConfig, + NotionContentConfig, + SearchConfig, + SearchResponse, +) from khoj.utils.state import SearchType # Initialize Router diff --git a/src/khoj/routers/api_beta.py b/src/khoj/routers/api_beta.py index 83cca612..0d87ec62 100644 --- a/src/khoj/routers/api_beta.py +++ b/src/khoj/routers/api_beta.py @@ -1,7 +1,5 @@ -# Standard Packages import logging -# External Packages from fastapi import APIRouter # Initialize Router diff --git a/src/khoj/routers/auth.py b/src/khoj/routers/auth.py index df119548..c7296316 100644 --- a/src/khoj/routers/auth.py +++ b/src/khoj/routers/auth.py @@ -1,26 +1,26 @@ -# Standard Packages import logging import os from typing import Optional -# External Packages +from authlib.integrations.starlette_client import OAuth, OAuthError from fastapi import APIRouter +from google.auth.transport import requests as google_requests +from google.oauth2 import id_token +from starlette.authentication import requires from starlette.config import Config from starlette.requests import Request from starlette.responses import HTMLResponse, RedirectResponse, Response -from starlette.authentication import requires -from authlib.integrations.starlette_client import OAuth, OAuthError -from google.oauth2 import id_token -from google.auth.transport import requests as google_requests - -# Internal Packages -from khoj.database.adapters import get_khoj_tokens, get_or_create_user, create_khoj_token, delete_khoj_token +from khoj.database.adapters import ( + create_khoj_token, + delete_khoj_token, + get_khoj_tokens, + get_or_create_user, +) from khoj.database.models import KhojApiUser from khoj.routers.helpers import update_telemetry_state from khoj.utils import state - logger = logging.getLogger(__name__) auth_router = APIRouter() diff --git a/src/khoj/routers/helpers.py b/src/khoj/routers/helpers.py index ff063cab..104426a3 100644 --- a/src/khoj/routers/helpers.py +++ b/src/khoj/routers/helpers.py @@ -1,4 +1,3 @@ -# Standard Packages import asyncio import json import logging @@ -9,23 +8,26 @@ from functools import partial from time import time from typing import Annotated, Any, Dict, Iterator, List, Optional, Tuple, Union -# External Packages -from fastapi import Depends, Header, HTTPException, Request, UploadFile import openai +from fastapi import Depends, Header, HTTPException, Request, UploadFile from starlette.authentication import has_required_scope -# Internal Packages from khoj.database.adapters import ConversationAdapters, EntryAdapters from khoj.database.models import KhojUser, Subscription, TextToImageModelConfig from khoj.processor.conversation import prompts -from khoj.processor.conversation.offline.chat_model import converse_offline, send_message_to_model_offline +from khoj.processor.conversation.offline.chat_model import ( + converse_offline, + send_message_to_model_offline, +) from khoj.processor.conversation.openai.gpt import converse, send_message_to_model -from khoj.processor.conversation.utils import ThreadedGenerator, save_to_conversation_log +from khoj.processor.conversation.utils import ( + ThreadedGenerator, + save_to_conversation_log, +) from khoj.utils import state from khoj.utils.config import GPT4AllProcessorModel from khoj.utils.helpers import ConversationCommand, log_telemetry - logger = logging.getLogger(__name__) executor = ThreadPoolExecutor(max_workers=1) diff --git a/src/khoj/routers/indexer.py b/src/khoj/routers/indexer.py index 8457d144..c7d4c01d 100644 --- a/src/khoj/routers/indexer.py +++ b/src/khoj/routers/indexer.py @@ -2,7 +2,7 @@ import asyncio import logging from typing import Dict, Optional, Union -from fastapi import APIRouter, Header, Request, Response, UploadFile, Depends +from fastapi import APIRouter, Depends, Header, Request, Response, UploadFile from pydantic import BaseModel from starlette.authentication import requires @@ -13,12 +13,11 @@ from khoj.processor.content.notion.notion_to_entries import NotionToEntries from khoj.processor.content.org_mode.org_to_entries import OrgToEntries from khoj.processor.content.pdf.pdf_to_entries import PdfToEntries from khoj.processor.content.plaintext.plaintext_to_entries import PlaintextToEntries -from khoj.routers.helpers import update_telemetry_state +from khoj.routers.helpers import ApiIndexedDataLimiter, update_telemetry_state from khoj.search_type import image_search, text_search from khoj.utils import constants, state from khoj.utils.config import ContentIndex, SearchModels from khoj.utils.helpers import LRU, get_file_type -from khoj.routers.helpers import ApiIndexedDataLimiter from khoj.utils.rawconfig import ContentConfig, FullConfig, SearchConfig from khoj.utils.yaml import save_config_to_file_updated_state diff --git a/src/khoj/routers/subscription.py b/src/khoj/routers/subscription.py index edcbc135..580fb225 100644 --- a/src/khoj/routers/subscription.py +++ b/src/khoj/routers/subscription.py @@ -1,18 +1,14 @@ -# Standard Packages -from datetime import datetime, timezone import logging import os +from datetime import datetime, timezone -# External Packages +import stripe from asgiref.sync import sync_to_async from fastapi import APIRouter, Request from starlette.authentication import requires -import stripe -# Internal Packages from khoj.database import adapters - # Stripe integration for Khoj Cloud Subscription stripe.api_key = os.getenv("STRIPE_API_KEY") endpoint_secret = os.getenv("STRIPE_SIGNING_SECRET") diff --git a/src/khoj/routers/web_client.py b/src/khoj/routers/web_client.py index 00a087b1..21d804ec 100644 --- a/src/khoj/routers/web_client.py +++ b/src/khoj/routers/web_client.py @@ -1,33 +1,30 @@ # System Packages import json -import os import math +import os from datetime import timedelta -# External Packages -from fastapi import APIRouter -from fastapi import Request -from fastapi.responses import HTMLResponse, FileResponse, RedirectResponse +from fastapi import APIRouter, Request +from fastapi.responses import FileResponse, HTMLResponse, RedirectResponse from fastapi.templating import Jinja2Templates -from starlette.authentication import requires, has_required_scope +from starlette.authentication import has_required_scope, requires + from khoj.database import adapters +from khoj.database.adapters import ( + ConversationAdapters, + EntryAdapters, + get_user_github_config, + get_user_notion_config, + get_user_subscription_state, +) from khoj.database.models import KhojUser +from khoj.utils import constants, state from khoj.utils.rawconfig import ( GithubContentConfig, GithubRepoConfig, NotionContentConfig, ) -# Internal Packages -from khoj.utils import constants, state -from khoj.database.adapters import ( - EntryAdapters, - get_user_github_config, - get_user_notion_config, - ConversationAdapters, - get_user_subscription_state, -) - # Initialize Router web_client = APIRouter() templates = Jinja2Templates(directory=constants.web_directory) diff --git a/src/khoj/search_filter/base_filter.py b/src/khoj/search_filter/base_filter.py index ae596587..ee6b10a5 100644 --- a/src/khoj/search_filter/base_filter.py +++ b/src/khoj/search_filter/base_filter.py @@ -1,4 +1,3 @@ -# Standard Packages from abc import ABC, abstractmethod from typing import List diff --git a/src/khoj/search_filter/date_filter.py b/src/khoj/search_filter/date_filter.py index 1d90b9f5..85b77076 100644 --- a/src/khoj/search_filter/date_filter.py +++ b/src/khoj/search_filter/date_filter.py @@ -1,20 +1,16 @@ -# Standard Packages -import re import logging +import re from collections import defaultdict -from datetime import timedelta, datetime -from typing import List -from dateutil.relativedelta import relativedelta +from datetime import datetime, timedelta from math import inf +from typing import List -# External Packages import dateparser as dtparse +from dateutil.relativedelta import relativedelta -# Internal Packages from khoj.search_filter.base_filter import BaseFilter from khoj.utils.helpers import LRU, timer - logger = logging.getLogger(__name__) diff --git a/src/khoj/search_filter/file_filter.py b/src/khoj/search_filter/file_filter.py index 291838ea..b3c3e0b8 100644 --- a/src/khoj/search_filter/file_filter.py +++ b/src/khoj/search_filter/file_filter.py @@ -1,15 +1,12 @@ -# Standard Packages -import re import fnmatch import logging +import re from collections import defaultdict from typing import List -# Internal Packages from khoj.search_filter.base_filter import BaseFilter from khoj.utils.helpers import LRU, timer - logger = logging.getLogger(__name__) diff --git a/src/khoj/search_filter/word_filter.py b/src/khoj/search_filter/word_filter.py index b2053dbe..d4d2e007 100644 --- a/src/khoj/search_filter/word_filter.py +++ b/src/khoj/search_filter/word_filter.py @@ -1,14 +1,11 @@ -# Standard Packages -import re import logging +import re from collections import defaultdict from typing import List -# Internal Packages from khoj.search_filter.base_filter import BaseFilter from khoj.utils.helpers import LRU - logger = logging.getLogger(__name__) diff --git a/src/khoj/search_type/image_search.py b/src/khoj/search_type/image_search.py index fd5107ad..76a72538 100644 --- a/src/khoj/search_type/image_search.py +++ b/src/khoj/search_type/image_search.py @@ -1,25 +1,27 @@ -# Standard Packages +import copy import glob +import logging import math import pathlib -import copy import shutil -import logging from typing import List -# External Packages -from sentence_transformers import SentenceTransformer, util -from PIL import Image -from tqdm import trange import torch +from PIL import Image +from sentence_transformers import SentenceTransformer, util +from tqdm import trange -# Internal Packages -from khoj.utils.helpers import get_absolute_path, get_from_dict, resolve_absolute_path, load_model, timer from khoj.utils.config import ImageContent, ImageSearchModel +from khoj.utils.helpers import ( + get_absolute_path, + get_from_dict, + load_model, + resolve_absolute_path, + timer, +) from khoj.utils.models import BaseEncoder from khoj.utils.rawconfig import ImageContentConfig, ImageSearchConfig, SearchResponse - # Create Logger logger = logging.getLogger(__name__) diff --git a/src/khoj/search_type/text_search.py b/src/khoj/search_type/text_search.py index 2111a8a7..58547ca5 100644 --- a/src/khoj/search_type/text_search.py +++ b/src/khoj/search_type/text_search.py @@ -1,26 +1,22 @@ -# Standard Packages import logging import math from pathlib import Path from typing import List, Tuple, Type, Union -# External Packages import torch +from asgiref.sync import sync_to_async from sentence_transformers import util -from asgiref.sync import sync_to_async - - -# Internal Packages +from khoj.database.adapters import EntryAdapters, get_user_search_model_or_default +from khoj.database.models import Entry as DbEntry +from khoj.database.models import KhojUser +from khoj.processor.content.text_to_entries import TextToEntries from khoj.utils import state from khoj.utils.helpers import get_absolute_path, timer -from khoj.utils.models import BaseEncoder -from khoj.utils.state import SearchType -from khoj.utils.rawconfig import SearchResponse, Entry from khoj.utils.jsonl import load_jsonl -from khoj.processor.content.text_to_entries import TextToEntries -from khoj.database.adapters import EntryAdapters, get_user_search_model_or_default -from khoj.database.models import KhojUser, Entry as DbEntry +from khoj.utils.models import BaseEncoder +from khoj.utils.rawconfig import Entry, SearchResponse +from khoj.utils.state import SearchType logger = logging.getLogger(__name__) diff --git a/src/khoj/utils/cli.py b/src/khoj/utils/cli.py index 2e4c9d59..968232e6 100644 --- a/src/khoj/utils/cli.py +++ b/src/khoj/utils/cli.py @@ -1,21 +1,23 @@ -# Standard Packages import argparse +import logging +import os import pathlib from importlib.metadata import version -import os -import logging logger = logging.getLogger(__name__) -# Internal Packages +from khoj.migrations.migrate_offline_chat_default_model import ( + migrate_offline_chat_default_model, +) +from khoj.migrations.migrate_offline_chat_schema import migrate_offline_chat_schema +from khoj.migrations.migrate_offline_model import migrate_offline_model +from khoj.migrations.migrate_processor_config_openai import ( + migrate_processor_conversation_schema, +) +from khoj.migrations.migrate_server_pg import migrate_server_pg +from khoj.migrations.migrate_version import migrate_config_to_version from khoj.utils.helpers import resolve_absolute_path from khoj.utils.yaml import parse_config_from_file -from khoj.migrations.migrate_version import migrate_config_to_version -from khoj.migrations.migrate_processor_config_openai import migrate_processor_conversation_schema -from khoj.migrations.migrate_offline_model import migrate_offline_model -from khoj.migrations.migrate_offline_chat_schema import migrate_offline_chat_schema -from khoj.migrations.migrate_offline_chat_default_model import migrate_offline_chat_default_model -from khoj.migrations.migrate_server_pg import migrate_server_pg def cli(args=None): diff --git a/src/khoj/utils/config.py b/src/khoj/utils/config.py index abda12b6..79a8957e 100644 --- a/src/khoj/utils/config.py +++ b/src/khoj/utils/config.py @@ -1,24 +1,21 @@ # System Packages from __future__ import annotations # to avoid quoting type hints -from enum import Enum import logging - from dataclasses import dataclass -from typing import TYPE_CHECKING, List, Optional, Union, Any +from enum import Enum +from typing import TYPE_CHECKING, Any, List, Optional, Union -# External Packages import torch -# Internal Packages from khoj.processor.conversation.offline.utils import download_model - logger = logging.getLogger(__name__) -# Internal Packages + if TYPE_CHECKING: from sentence_transformers import CrossEncoder + from khoj.utils.models import BaseEncoder diff --git a/src/khoj/utils/fs_syncer.py b/src/khoj/utils/fs_syncer.py index 57c79d9d..31bc13b9 100644 --- a/src/khoj/utils/fs_syncer.py +++ b/src/khoj/utils/fs_syncer.py @@ -1,13 +1,19 @@ -import logging import glob +import logging import os from typing import Optional + from bs4 import BeautifulSoup +from khoj.database.models import ( + LocalMarkdownConfig, + LocalOrgConfig, + LocalPdfConfig, + LocalPlaintextConfig, +) +from khoj.utils.config import SearchType from khoj.utils.helpers import get_absolute_path, is_none_or_empty from khoj.utils.rawconfig import TextContentConfig -from khoj.utils.config import SearchType -from khoj.database.models import LocalMarkdownConfig, LocalOrgConfig, LocalPdfConfig, LocalPlaintextConfig logger = logging.getLogger(__name__) diff --git a/src/khoj/utils/helpers.py b/src/khoj/utils/helpers.py index 21fe7e98..df2a3cd0 100644 --- a/src/khoj/utils/helpers.py +++ b/src/khoj/utils/helpers.py @@ -1,32 +1,29 @@ -# Standard Packages from __future__ import annotations # to avoid quoting type hints -from collections import OrderedDict + import datetime +import logging +import os +import platform +import random +import uuid +from collections import OrderedDict from enum import Enum from importlib import import_module from importlib.metadata import version from itertools import islice -import logging from os import path -import os from pathlib import Path -import platform -import random from time import perf_counter +from typing import TYPE_CHECKING, Optional, Union + import torch -from typing import Optional, Union, TYPE_CHECKING -import uuid from asgiref.sync import sync_to_async -# Internal Packages from khoj.utils import constants - if TYPE_CHECKING: - # External Packages - from sentence_transformers import SentenceTransformer, CrossEncoder + from sentence_transformers import CrossEncoder, SentenceTransformer - # Internal Packages from khoj.utils.models import BaseEncoder from khoj.utils.rawconfig import AppConfig diff --git a/src/khoj/utils/initialization.py b/src/khoj/utils/initialization.py index 0bb78dbe..abee1635 100644 --- a/src/khoj/utils/initialization.py +++ b/src/khoj/utils/initialization.py @@ -1,20 +1,17 @@ import logging import os +from khoj.database.adapters import ConversationAdapters from khoj.database.models import ( + ChatModelOptions, KhojUser, OfflineChatProcessorConversationConfig, OpenAIProcessorConversationConfig, - ChatModelOptions, SpeechToTextModelOptions, TextToImageModelConfig, ) - -from khoj.utils.constants import default_offline_chat_model, default_online_chat_model from khoj.processor.conversation.utils import model_to_prompt_size, model_to_tokenizer - -from khoj.database.adapters import ConversationAdapters - +from khoj.utils.constants import default_offline_chat_model, default_online_chat_model logger = logging.getLogger(__name__) diff --git a/src/khoj/utils/jsonl.py b/src/khoj/utils/jsonl.py index ed779e79..d01c3c19 100644 --- a/src/khoj/utils/jsonl.py +++ b/src/khoj/utils/jsonl.py @@ -1,13 +1,10 @@ -# Standard Packages -import json import gzip +import json import logging -# Internal Packages from khoj.utils.constants import empty_escape_sequences from khoj.utils.helpers import get_absolute_path - logger = logging.getLogger(__name__) diff --git a/src/khoj/utils/models.py b/src/khoj/utils/models.py index e1298b08..b76d369a 100644 --- a/src/khoj/utils/models.py +++ b/src/khoj/utils/models.py @@ -1,13 +1,10 @@ -# Standard Packages from abc import ABC, abstractmethod from typing import List -# External Packages import openai import torch from tqdm import trange -# Internal Packages from khoj.utils import state diff --git a/src/khoj/utils/rawconfig.py b/src/khoj/utils/rawconfig.py index d36a36ff..67b0747c 100644 --- a/src/khoj/utils/rawconfig.py +++ b/src/khoj/utils/rawconfig.py @@ -1,13 +1,11 @@ # System Packages import json -from pathlib import Path -from typing import List, Dict, Optional import uuid +from pathlib import Path +from typing import Dict, List, Optional -# External Packages from pydantic import BaseModel -# Internal Packages from khoj.utils.helpers import to_snake_case_from_dash diff --git a/src/khoj/utils/state.py b/src/khoj/utils/state.py index 03fdd52f..3a1167a9 100644 --- a/src/khoj/utils/state.py +++ b/src/khoj/utils/state.py @@ -1,18 +1,15 @@ -# Standard Packages -from collections import defaultdict import os -from pathlib import Path import threading -from typing import List, Dict +from collections import defaultdict +from pathlib import Path +from typing import Dict, List -# External Packages from openai import OpenAI from whisper import Whisper -# Internal Packages from khoj.processor.embeddings import CrossEncoderModel, EmbeddingsModel from khoj.utils import config as utils_config -from khoj.utils.config import ContentIndex, SearchModels, GPT4AllProcessorModel +from khoj.utils.config import ContentIndex, GPT4AllProcessorModel, SearchModels from khoj.utils.helpers import LRU, get_device from khoj.utils.rawconfig import FullConfig diff --git a/src/khoj/utils/yaml.py b/src/khoj/utils/yaml.py index 36546688..e4bf17f3 100644 --- a/src/khoj/utils/yaml.py +++ b/src/khoj/utils/yaml.py @@ -1,13 +1,9 @@ -# Standard Packages from pathlib import Path -# External Packages import yaml -# Internal Packages -from khoj.utils.rawconfig import FullConfig from khoj.utils import state - +from khoj.utils.rawconfig import FullConfig # Do not emit tags when dumping to YAML yaml.emitter.Emitter.process_tag = lambda self, *args, **kwargs: None # type: ignore[assignment] diff --git a/src/telemetry/telemetry.py b/src/telemetry/telemetry.py index fabaafa0..576befd0 100644 --- a/src/telemetry/telemetry.py +++ b/src/telemetry/telemetry.py @@ -1,17 +1,13 @@ -# Standard Packages import argparse import logging import os +import sqlite3 from typing import Dict, List -# External Packages -from fastapi import FastAPI -from fastapi import HTTPException -from posthog import Posthog -from dotenv import load_dotenv -import sqlite3 import uvicorn - +from dotenv import load_dotenv +from fastapi import FastAPI, HTTPException +from posthog import Posthog # Initialize Global App Variables app = FastAPI() diff --git a/tests/conftest.py b/tests/conftest.py index bbb3aa39..f7756d99 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,3 @@ -# External Packages import os from pathlib import Path @@ -7,8 +6,11 @@ from fastapi import FastAPI from fastapi.staticfiles import StaticFiles from fastapi.testclient import TestClient -# Internal Packages -from khoj.configure import configure_middleware, configure_routes, configure_search_types +from khoj.configure import ( + configure_middleware, + configure_routes, + configure_search_types, +) from khoj.database.models import ( GithubConfig, GithubRepoConfig, @@ -27,7 +29,12 @@ from khoj.utils import fs_syncer, state from khoj.utils.config import SearchModels from khoj.utils.constants import web_directory from khoj.utils.helpers import resolve_absolute_path -from khoj.utils.rawconfig import ContentConfig, ImageContentConfig, ImageSearchConfig, SearchConfig +from khoj.utils.rawconfig import ( + ContentConfig, + ImageContentConfig, + ImageSearchConfig, + SearchConfig, +) from tests.helpers import ( ChatModelOptionsFactory, OfflineChatProcessorConversationConfigFactory, diff --git a/tests/helpers.py b/tests/helpers.py index d0f9babc..cc0f1ed3 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,16 +1,17 @@ -import factory import os +import factory + from khoj.database.models import ( - KhojUser, - KhojApiUser, ChatModelOptions, + Conversation, + KhojApiUser, + KhojUser, OfflineChatProcessorConversationConfig, OpenAIProcessorConversationConfig, SearchModelConfig, - UserConversationConfig, - Conversation, Subscription, + UserConversationConfig, ) diff --git a/tests/test_cli.py b/tests/test_cli.py index e3daa2c6..211ff38e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2,7 +2,6 @@ from pathlib import Path from random import random -# Internal Packages from khoj.utils.cli import cli from khoj.utils.helpers import resolve_absolute_path diff --git a/tests/test_client.py b/tests/test_client.py index f23a350e..0bc3c02f 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -4,12 +4,9 @@ from urllib.parse import quote import pytest from fastapi import FastAPI - -# External Packages from fastapi.testclient import TestClient from PIL import Image -# Internal Packages from khoj.configure import configure_routes, configure_search_types from khoj.database.adapters import EntryAdapters from khoj.database.models import KhojApiUser, KhojUser diff --git a/tests/test_conversation_utils.py b/tests/test_conversation_utils.py index 3cfddc90..52db0002 100644 --- a/tests/test_conversation_utils.py +++ b/tests/test_conversation_utils.py @@ -1,7 +1,8 @@ -from khoj.processor.conversation import utils -from langchain.schema import ChatMessage import factory import tiktoken +from langchain.schema import ChatMessage + +from khoj.processor.conversation import utils class ChatMessageFactory(factory.Factory): diff --git a/tests/test_date_filter.py b/tests/test_date_filter.py index f1f26d28..adb6c2c9 100644 --- a/tests/test_date_filter.py +++ b/tests/test_date_filter.py @@ -1,11 +1,8 @@ -# Standard Packages import re from datetime import datetime -# External Packages import pytest -# Internal Packages from khoj.search_filter.date_filter import DateFilter diff --git a/tests/test_gpt4all_chat_actors.py b/tests/test_gpt4all_chat_actors.py index 7b59e1e3..30209b81 100644 --- a/tests/test_gpt4all_chat_actors.py +++ b/tests/test_gpt4all_chat_actors.py @@ -1,7 +1,5 @@ -# Standard Packages from datetime import datetime -# External Packages import pytest SKIP_TESTS = True @@ -18,10 +16,13 @@ try: except ModuleNotFoundError as e: print("There was an error importing GPT4All. Please run pip install gpt4all in order to install it.") -# Internal Packages -from khoj.processor.conversation.offline.chat_model import converse_offline, extract_questions_offline, filter_questions -from khoj.processor.conversation.offline.utils import download_model +from khoj.processor.conversation.offline.chat_model import ( + converse_offline, + extract_questions_offline, + filter_questions, +) +from khoj.processor.conversation.offline.utils import download_model from khoj.processor.conversation.utils import message_to_log MODEL_NAME = "mistral-7b-instruct-v0.1.Q4_0.gguf" diff --git a/tests/test_gpt4all_chat_director.py b/tests/test_gpt4all_chat_director.py index d44a3c07..28bc3a8f 100644 --- a/tests/test_gpt4all_chat_director.py +++ b/tests/test_gpt4all_chat_director.py @@ -1,13 +1,10 @@ -# Standard Packages import urllib.parse from urllib.parse import quote -# External Packages import pytest -from freezegun import freeze_time from faker import Faker +from freezegun import freeze_time -# Internal Packages from khoj.processor.conversation import prompts from khoj.processor.conversation.utils import message_to_log from tests.helpers import ConversationFactory diff --git a/tests/test_helpers.py b/tests/test_helpers.py index fdd29b02..79cf1e9d 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -1,13 +1,10 @@ -# Standard Packages -import numpy as np -import psutil -from scipy.stats import linregress import secrets -# External Packages +import numpy as np +import psutil import pytest +from scipy.stats import linregress -# Internal Packages from khoj.processor.embeddings import EmbeddingsModel from khoj.utils import helpers diff --git a/tests/test_image_search.py b/tests/test_image_search.py index 82617ab3..5fe9ac7a 100644 --- a/tests/test_image_search.py +++ b/tests/test_image_search.py @@ -1,18 +1,16 @@ # Standard Modules import logging from pathlib import Path + +import pytest from PIL import Image -# External Packages -import pytest -from khoj.utils.config import SearchModels - -# Internal Packages -from khoj.utils.state import content_index, search_models -from khoj.utils.constants import web_directory from khoj.search_type import image_search +from khoj.utils.config import SearchModels +from khoj.utils.constants import web_directory from khoj.utils.helpers import resolve_absolute_path from khoj.utils.rawconfig import ContentConfig, SearchConfig +from khoj.utils.state import content_index, search_models # Test diff --git a/tests/test_markdown_to_entries.py b/tests/test_markdown_to_entries.py index b20040f9..4a4a75f3 100644 --- a/tests/test_markdown_to_entries.py +++ b/tests/test_markdown_to_entries.py @@ -1,9 +1,7 @@ -# Standard Packages import json -from pathlib import Path import os +from pathlib import Path -# Internal Packages from khoj.processor.content.markdown.markdown_to_entries import MarkdownToEntries from khoj.utils.fs_syncer import get_markdown_files from khoj.utils.rawconfig import TextContentConfig diff --git a/tests/test_multiple_users.py b/tests/test_multiple_users.py index dae881bf..bb0f99d8 100644 --- a/tests/test_multiple_users.py +++ b/tests/test_multiple_users.py @@ -1,13 +1,10 @@ # Standard Modules from urllib.parse import quote -# External Packages import pytest from khoj.database.models import KhojApiUser, KhojUser from khoj.processor.content.org_mode.org_to_entries import OrgToEntries - -# Internal Packages from khoj.search_type import text_search diff --git a/tests/test_openai_chat_actors.py b/tests/test_openai_chat_actors.py index e84f41f7..46f63d0e 100644 --- a/tests/test_openai_chat_actors.py +++ b/tests/test_openai_chat_actors.py @@ -1,17 +1,13 @@ -# Standard Packages import os from datetime import datetime -# External Packages -import pytest import freezegun +import pytest from freezegun import freeze_time -# Internal Packages from khoj.processor.conversation.openai.gpt import converse, extract_questions from khoj.processor.conversation.utils import message_to_log - # Initialize variables for tests api_key = os.getenv("OPENAI_API_KEY") if api_key is None: diff --git a/tests/test_openai_chat_director.py b/tests/test_openai_chat_director.py index 50ad60af..181080ef 100644 --- a/tests/test_openai_chat_director.py +++ b/tests/test_openai_chat_director.py @@ -1,17 +1,14 @@ -# Standard Packages import os import urllib.parse from urllib.parse import quote -# External Packages import pytest from freezegun import freeze_time -from khoj.processor.conversation import prompts -# Internal Packages +from khoj.database.models import KhojUser +from khoj.processor.conversation import prompts from khoj.processor.conversation.utils import message_to_log from tests.helpers import ConversationFactory -from khoj.database.models import KhojUser # Initialize variables for tests api_key = os.getenv("OPENAI_API_KEY") diff --git a/tests/test_org_to_entries.py b/tests/test_org_to_entries.py index 6cd5e3ce..f7c19b79 100644 --- a/tests/test_org_to_entries.py +++ b/tests/test_org_to_entries.py @@ -1,14 +1,11 @@ -# Standard Packages import json import os -# Internal Packages from khoj.processor.content.org_mode.org_to_entries import OrgToEntries from khoj.processor.content.text_to_entries import TextToEntries -from khoj.utils.helpers import is_none_or_empty -from khoj.utils.rawconfig import Entry from khoj.utils.fs_syncer import get_org_files -from khoj.utils.rawconfig import TextContentConfig +from khoj.utils.helpers import is_none_or_empty +from khoj.utils.rawconfig import Entry, TextContentConfig def test_configure_heading_entry_to_jsonl(tmp_path): diff --git a/tests/test_orgnode.py b/tests/test_orgnode.py index a543ceb5..49344325 100644 --- a/tests/test_orgnode.py +++ b/tests/test_orgnode.py @@ -1,7 +1,5 @@ -# Standard Packages import datetime -# Internal Packages from khoj.processor.content.org_mode import orgnode diff --git a/tests/test_pdf_to_entries.py b/tests/test_pdf_to_entries.py index 9aeb438f..62decdd7 100644 --- a/tests/test_pdf_to_entries.py +++ b/tests/test_pdf_to_entries.py @@ -1,10 +1,7 @@ -# Standard Packages import json import os -# Internal Packages from khoj.processor.content.pdf.pdf_to_entries import PdfToEntries - from khoj.utils.fs_syncer import get_pdf_files from khoj.utils.rawconfig import TextContentConfig diff --git a/tests/test_plaintext_to_entries.py b/tests/test_plaintext_to_entries.py index d3d50c94..53585177 100644 --- a/tests/test_plaintext_to_entries.py +++ b/tests/test_plaintext_to_entries.py @@ -1,12 +1,9 @@ -# Standard Packages import json import os from pathlib import Path from khoj.database.models import KhojUser, LocalPlaintextConfig from khoj.processor.content.plaintext.plaintext_to_entries import PlaintextToEntries - -# Internal Packages from khoj.utils.fs_syncer import get_plaintext_files from khoj.utils.rawconfig import TextContentConfig diff --git a/tests/test_rawconfig.py b/tests/test_rawconfig.py index 3f3028b5..6598f25b 100644 --- a/tests/test_rawconfig.py +++ b/tests/test_rawconfig.py @@ -1,8 +1,6 @@ -# External Packages import pytest -# Internal Packages -from khoj.utils.rawconfig import TextContentConfig, ImageContentConfig +from khoj.utils.rawconfig import ImageContentConfig, TextContentConfig # Test diff --git a/tests/test_text_search.py b/tests/test_text_search.py index 186ef57c..791ce91b 100644 --- a/tests/test_text_search.py +++ b/tests/test_text_search.py @@ -4,14 +4,11 @@ import logging import os from pathlib import Path -# External Packages import pytest from khoj.database.models import Entry, GithubConfig, KhojUser, LocalOrgConfig from khoj.processor.content.github.github_to_entries import GithubToEntries from khoj.processor.content.org_mode.org_to_entries import OrgToEntries - -# Internal Packages from khoj.search_type import text_search from khoj.utils.fs_syncer import collect_files, get_org_files from khoj.utils.rawconfig import ContentConfig, SearchConfig