mirror of
https://github.com/khoj-ai/khoj.git
synced 2024-11-23 23:48:56 +01:00
ecc6fbfeb2
### Overview
- Add ability to push data to index from the Emacs, Obsidian client
- Switch to standard mechanism of syncing files via HTTP multi-part/form. Previously we were streaming the data as JSON
- Benefits of new mechanism
- No manual parsing of files to send or receive on clients or server is required as most have in-built mechanisms to send multi-part/form requests
- The whole response is not required to be kept in memory to parse content as JSON. As individual files arrive they're automatically pushed to disk to conserve memory if required
- Binary files don't need to be encoded on client and decoded on server
### Code Details
### Major
- Use multi-part form to receive files to index on server
- Use multi-part form to send files to index on desktop client
- Send files to index on server from the khoj.el emacs client
- Send content for indexing on server at a regular interval from khoj.el
- Send files to index on server from the khoj obsidian client
- Update tests to test multi-part/form method of pushing files to index
#### Minor
- Put indexer API endpoint under /api path segment
- Explicitly make GET request to /config/data from khoj.el:khoj-server-configure method
- Improve emoji, message on content index updated via logger
- Don't call khoj server on khoj.el load, only once khoj invoked explicitly by user
- Improve indexing of binary files
- Let fs_syncer pass PDF files directly as binary before indexing
- Use encoding of each file set in indexer request to read file
- Add CORS policy to khoj server. Allow requests from khoj apps, obsidian & localhost
- Update indexer API endpoint URL to` index/update` from `indexer/batch`
Resolves #471 #243
119 lines
3 KiB
TOML
119 lines
3 KiB
TOML
[build-system]
|
|
requires = ["hatchling", "hatch-vcs"]
|
|
build-backend = "hatchling.build"
|
|
|
|
[project]
|
|
name = "khoj-assistant"
|
|
description = "An AI copilot for your Second Brain"
|
|
readme = "README.md"
|
|
license = "GPL-3.0-or-later"
|
|
requires-python = ">=3.8"
|
|
authors = [
|
|
{ name = "Debanjum Singh Solanky, Saba Imran" },
|
|
]
|
|
keywords = [
|
|
"search",
|
|
"semantic-search",
|
|
"productivity",
|
|
"NLP",
|
|
"AI",
|
|
"org-mode",
|
|
"markdown",
|
|
"images",
|
|
"pdf",
|
|
]
|
|
classifiers = [
|
|
"Development Status :: 4 - Beta",
|
|
"License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
|
|
"Operating System :: OS Independent",
|
|
"Programming Language :: Python :: 3",
|
|
"Programming Language :: Python :: 3.9",
|
|
"Programming Language :: Python :: 3.10",
|
|
"Programming Language :: Python :: 3.11",
|
|
"Topic :: Internet :: WWW/HTTP :: Indexing/Search",
|
|
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
"Topic :: Scientific/Engineering :: Human Machine Interfaces",
|
|
"Topic :: Text Processing :: Linguistic",
|
|
]
|
|
dependencies = [
|
|
"bs4 >= 0.0.1",
|
|
"dateparser >= 1.1.1",
|
|
"defusedxml == 0.7.1",
|
|
"fastapi == 0.77.1",
|
|
"python-multipart >= 0.0.5",
|
|
"jinja2 == 3.1.2",
|
|
"openai >= 0.27.0, < 1.0.0",
|
|
"tiktoken >= 0.3.2",
|
|
"tenacity >= 8.2.2",
|
|
"pillow == 9.3.0",
|
|
"pydantic >= 1.10.10",
|
|
"pyyaml == 6.0",
|
|
"rich >= 13.3.1",
|
|
"schedule == 1.1.0",
|
|
"sentence-transformers == 2.2.2",
|
|
"transformers >= 4.28.0",
|
|
"torch >= 2.0.1",
|
|
"uvicorn == 0.17.6",
|
|
"aiohttp == 3.8.5",
|
|
"langchain >= 0.0.187",
|
|
"requests >= 2.26.0",
|
|
"bs4 >= 0.0.1",
|
|
"anyio == 3.7.1",
|
|
"pymupdf >= 1.23.3",
|
|
"gpt4all == 1.0.12; platform_system == 'Linux' and platform_machine == 'x86_64'",
|
|
"gpt4all == 1.0.12; platform_system == 'Windows' or platform_system == 'Darwin'",
|
|
]
|
|
dynamic = ["version"]
|
|
|
|
[project.urls]
|
|
Homepage = "https://github.com/khoj-ai/khoj#readme"
|
|
Issues = "https://github.com/khoj-ai/khoj/issues"
|
|
Discussions = "https://github.com/khoj-ai/khoj/discussions"
|
|
Releases = "https://github.com/khoj-ai/khoj/releases"
|
|
|
|
[project.scripts]
|
|
khoj = "khoj.main:run"
|
|
|
|
[project.optional-dependencies]
|
|
test = [
|
|
"pytest >= 7.1.2",
|
|
"freezegun >= 1.2.0",
|
|
"factory-boy >= 3.2.1",
|
|
"trio >= 0.22.0",
|
|
"pytest-xdist",
|
|
]
|
|
dev = [
|
|
"khoj-assistant[test]",
|
|
"mypy >= 1.0.1",
|
|
"black >= 23.1.0",
|
|
"pre-commit >= 3.0.4",
|
|
]
|
|
|
|
[tool.hatch.version]
|
|
source = "vcs"
|
|
raw-options.local_scheme = "no-local-version" # PEP440 compliant version for PyPi
|
|
|
|
[tool.hatch.build.targets.sdist]
|
|
include = ["src/khoj"]
|
|
|
|
[tool.hatch.build.targets.wheel]
|
|
packages = ["src/khoj"]
|
|
|
|
[tool.mypy]
|
|
files = "src/khoj"
|
|
pretty = true
|
|
strict_optional = false
|
|
install_types = true
|
|
ignore_missing_imports = true
|
|
non_interactive = true
|
|
show_error_codes = true
|
|
warn_unused_ignores = false
|
|
|
|
[tool.black]
|
|
line-length = 120
|
|
|
|
[tool.pytest.ini_options]
|
|
addopts = "--strict-markers"
|
|
markers = [
|
|
"chatquality: Evaluate chatbot capabilities and quality",
|
|
]
|