sij/anything-llm
1
0
Fork 0
mirror of https://github.com/Mintplex-Labs/anything-llm.git synced 2025-05-02 09:03:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Github data connector improvements ()

Browse source 
* fix tree/blob github urls from branches not being loaded

* improve ux of github data connector

* lint

* patch Github URL parser to just validate with `URL` native parser

* uncheck LocalStorage of PAT for security reasons

---------

Co-authored-by: Timothy Carambat <rambat1010@gmail.com>
This commit is contained in:
Sean Hatfield 2024-10-21 15:25:35 -07:00 • committed by GitHub
parent c734742189
commit 0074ededdd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 29 additions and 13 deletions

42
collector/utils/extensions/RepoLoader/GithubRepo/RepoLoader/index.js
View file

@ -29,20 +29,36 @@ class GitHubRepoLoader {
}
#validGithubUrl() {
const UrlPattern = require("url-pattern");
const pattern = new UrlPattern(
"https\\://github.com/(:author)/(:project(*))",
{
// fixes project names with special characters (.github)
segmentValueCharset: "a-zA-Z0-9-._~%/+",
}
);
const match = pattern.match(this.repo);
if (!match) return false;
try {
const url = new URL(this.repo);
this.author = match.author;
this.project = match.project;
return true;
// Not a github url at all.
if (url.hostname !== "github.com") {
console.log(
`[Github Loader]: Invalid Github URL provided! Hostname must be 'github.com'. Got ${url.hostname}`
);
return false;
}
// Assume the url is in the format of github.com/{author}/{project}
// Remove the first slash from the pathname so we can split it properly.
const [author, project, ..._rest] = url.pathname.slice(1).split("/");
if (!author || !project) {
console.log(
`[Github Loader]: Invalid Github URL provided! URL must be in the format of 'github.com/{author}/{project}'. Got ${url.pathname}`
);
return false;
}
this.author = author;
this.project = project;
return true;
} catch (e) {
console.log(
`[Github Loader]: Invalid Github URL provided! Error: ${e.message}`
);
return false;
}
}
// Ensure the branch provided actually exists