diff --git a/cmd/flags.go b/cmd/flags.go
index 3c1ac99..8ac09ec 100644
--- a/cmd/flags.go
+++ b/cmd/flags.go
@@ -63,6 +63,19 @@ var ServeFlags = []cli.Flag{
// TODO: desc
EnvVars: []string{"ENABLE_HTTP_SERVER"},
},
+ // Server Options
+ &cli.BoolFlag{
+ Name: "enable-lfs-support",
+ Usage: "enable lfs support, require gitea v1.17.0 as backend",
+ EnvVars: []string{"ENABLE_LFS_SUPPORT"},
+ Value: true,
+ },
+ &cli.BoolFlag{
+ Name: "enable-symlink-support",
+ Usage: "follow symlinks if enabled, require gitea v1.18.0 as backend",
+ EnvVars: []string{"ENABLE_SYMLINK_SUPPORT"},
+ Value: true,
+ },
&cli.StringFlag{
Name: "log-level",
Value: "warn",
diff --git a/cmd/main.go b/cmd/main.go
index f57eb60..41809cb 100644
--- a/cmd/main.go
+++ b/cmd/main.go
@@ -85,7 +85,7 @@ func Serve(ctx *cli.Context) error {
// TODO: make this an MRU cache with a size limit
fileResponseCache := cache.NewKeyValueCache()
- giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken)
+ giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken, ctx.Bool("enable-symlink-support"), ctx.Bool("enable-lfs-support"))
if err != nil {
return fmt.Errorf("could not create new gitea client: %v", err)
}
diff --git a/go.mod b/go.mod
index 64288de..479c328 100644
--- a/go.mod
+++ b/go.mod
@@ -8,6 +8,7 @@ require (
github.com/go-acme/lego/v4 v4.5.3
github.com/joho/godotenv v1.4.0
github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad
+ github.com/rs/zerolog v1.27.0
github.com/stretchr/testify v1.7.0
github.com/urfave/cli/v2 v2.3.0
github.com/valyala/fasthttp v1.31.0
@@ -92,7 +93,6 @@ require (
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/pquerna/otp v1.3.0 // indirect
- github.com/rs/zerolog v1.27.0 // indirect
github.com/russross/blackfriday/v2 v2.0.1 // indirect
github.com/sacloud/libsacloud v1.36.2 // indirect
github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210127161313-bd30bebeac4f // indirect
diff --git a/go.sum b/go.sum
index 4ac6eb6..23a58bc 100644
--- a/go.sum
+++ b/go.sum
@@ -327,7 +327,6 @@ github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
-github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
@@ -671,8 +670,6 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA=
-golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 h1:foEbQz/B0Oz6YIqu/69kfXPYeFQAuuMYFkjaqXzl5Wo=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
diff --git a/integration/get_test.go b/integration/get_test.go
index 191fa3f..6054e17 100644
--- a/integration/get_test.go
+++ b/integration/get_test.go
@@ -10,6 +10,7 @@ import (
"log"
"net/http"
"net/http/cookiejar"
+ "strings"
"testing"
"github.com/stretchr/testify/assert"
@@ -88,6 +89,34 @@ func TestGetNotFound(t *testing.T) {
assert.EqualValues(t, 37, getSize(resp.Body))
}
+func TestFollowSymlink(t *testing.T) {
+ log.Printf("=== TestFollowSymlink ===\n")
+
+ resp, err := getTestHTTPSClient().Get("https://6543.localhost.mock.directory:4430/tests_for_pages-server/@main/link")
+ assert.NoError(t, err)
+ if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+ t.FailNow()
+ }
+ assert.EqualValues(t, "application/octet-stream", resp.Header.Get("Content-Type"))
+ assert.EqualValues(t, "4", resp.Header.Get("Content-Length"))
+ body := getBytes(resp.Body)
+ assert.EqualValues(t, 4, len(body))
+ assert.EqualValues(t, "abc\n", string(body))
+}
+
+func TestLFSSupport(t *testing.T) {
+ log.Printf("=== TestLFSSupport ===\n")
+
+ resp, err := getTestHTTPSClient().Get("https://6543.localhost.mock.directory:4430/tests_for_pages-server/@main/lfs.txt")
+ assert.NoError(t, err)
+ if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+ t.FailNow()
+ }
+ body := strings.TrimSpace(string(getBytes(resp.Body)))
+ assert.EqualValues(t, 12, len(body))
+ assert.EqualValues(t, "actual value", body)
+}
+
func getTestHTTPSClient() *http.Client {
cookieJar, _ := cookiejar.New(nil)
return &http.Client{
@@ -101,6 +130,12 @@ func getTestHTTPSClient() *http.Client {
}
}
+func getBytes(stream io.Reader) []byte {
+ buf := new(bytes.Buffer)
+ _, _ = buf.ReadFrom(stream)
+ return buf.Bytes()
+}
+
func getSize(stream io.Reader) int {
buf := new(bytes.Buffer)
_, _ = buf.ReadFrom(stream)
diff --git a/server/gitea/cache.go b/server/gitea/cache.go
new file mode 100644
index 0000000..932ff3c
--- /dev/null
+++ b/server/gitea/cache.go
@@ -0,0 +1,12 @@
+package gitea
+
+type FileResponse struct {
+ Exists bool
+ ETag []byte
+ MimeType string
+ Body []byte
+}
+
+func (f FileResponse) IsEmpty() bool {
+ return len(f.Body) != 0
+}
diff --git a/server/gitea/client.go b/server/gitea/client.go
index 3b9ad6f..16cba84 100644
--- a/server/gitea/client.go
+++ b/server/gitea/client.go
@@ -7,11 +7,15 @@ import (
"strings"
"time"
+ "github.com/rs/zerolog/log"
"github.com/valyala/fasthttp"
"github.com/valyala/fastjson"
)
-const giteaAPIRepos = "/api/v1/repos/"
+const (
+ giteaAPIRepos = "/api/v1/repos/"
+ giteaObjectTypeHeader = "X-Gitea-Object-Type"
+)
var ErrorNotFound = errors.New("not found")
@@ -21,13 +25,9 @@ type Client struct {
fastClient *fasthttp.Client
infoTimeout time.Duration
contentTimeout time.Duration
-}
-type FileResponse struct {
- Exists bool
- ETag []byte
- MimeType string
- Body []byte
+ followSymlinks bool
+ supportLFS bool
}
// TODO: once golang v1.19 is min requirement, we can switch to 'JoinPath()' of 'net/url' package
@@ -44,9 +44,7 @@ func joinURL(baseURL string, paths ...string) string {
return baseURL + "/" + strings.Join(p, "/")
}
-func (f FileResponse) IsEmpty() bool { return len(f.Body) != 0 }
-
-func NewClient(giteaRoot, giteaAPIToken string) (*Client, error) {
+func NewClient(giteaRoot, giteaAPIToken string, followSymlinks, supportLFS bool) (*Client, error) {
rootURL, err := url.Parse(giteaRoot)
giteaRoot = strings.Trim(rootURL.String(), "/")
@@ -56,29 +54,28 @@ func NewClient(giteaRoot, giteaAPIToken string) (*Client, error) {
infoTimeout: 5 * time.Second,
contentTimeout: 10 * time.Second,
fastClient: getFastHTTPClient(),
+
+ followSymlinks: followSymlinks,
+ supportLFS: supportLFS,
}, err
}
func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource string) ([]byte, error) {
- url := joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref))
- res, err := client.do(client.contentTimeout, url)
+ resp, err := client.ServeRawContent(targetOwner, targetRepo, ref, resource)
if err != nil {
return nil, err
}
-
- switch res.StatusCode() {
- case fasthttp.StatusOK:
- return res.Body(), nil
- case fasthttp.StatusNotFound:
- return nil, ErrorNotFound
- default:
- return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
- }
+ return resp.Body(), nil
}
-func (client *Client) ServeRawContent(uri string) (*fasthttp.Response, error) {
- url := joinURL(client.giteaRoot, giteaAPIRepos, uri)
- res, err := client.do(client.contentTimeout, url)
+func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource string) (*fasthttp.Response, error) {
+ var apiURL string
+ if client.supportLFS {
+ apiURL = joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "media", resource+"?ref="+url.QueryEscape(ref))
+ } else {
+ apiURL = joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref))
+ }
+ resp, err := client.do(client.contentTimeout, apiURL)
if err != nil {
return nil, err
}
@@ -87,13 +84,24 @@ func (client *Client) ServeRawContent(uri string) (*fasthttp.Response, error) {
return nil, err
}
- switch res.StatusCode() {
+ switch resp.StatusCode() {
case fasthttp.StatusOK:
- return res, nil
+ objType := string(resp.Header.Peek(giteaObjectTypeHeader))
+ log.Trace().Msgf("server raw content object: %s", objType)
+ if client.followSymlinks && objType == "symlink" {
+ // TODO: limit to 1000 chars if we switched to std
+ linkDest := strings.TrimSpace(string(resp.Body()))
+ log.Debug().Msgf("follow symlink from '%s' to '%s'", resource, linkDest)
+ return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest)
+ }
+
+ return resp, nil
+
case fasthttp.StatusNotFound:
return nil, ErrorNotFound
+
default:
- return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
+ return nil, fmt.Errorf("unexpected status code '%d'", resp.StatusCode())
}
}
diff --git a/server/handler_test.go b/server/handler_test.go
index 23d9af5..f9a721a 100644
--- a/server/handler_test.go
+++ b/server/handler_test.go
@@ -13,7 +13,7 @@ import (
func TestHandlerPerformance(t *testing.T) {
giteaRoot := "https://codeberg.org"
- giteaClient, _ := gitea.NewClient(giteaRoot, "")
+ giteaClient, _ := gitea.NewClient(giteaRoot, "", false, false)
testHandler := Handler(
[]byte("codeberg.page"), []byte("raw.codeberg.org"),
giteaClient,
diff --git a/server/upstream/helper.go b/server/upstream/helper.go
index 5bbe833..0714dcd 100644
--- a/server/upstream/helper.go
+++ b/server/upstream/helper.go
@@ -67,6 +67,10 @@ func (o *Options) generateUri() string {
return path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath)
}
+func (o *Options) generateUriClientArgs() (targetOwner, targetRepo, ref, resource string) {
+ return o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath
+}
+
func (o *Options) timestamp() string {
return strconv.FormatInt(o.BranchTimestamp.Unix(), 10)
}
diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go
index 4371e88..0e27727 100644
--- a/server/upstream/upstream.go
+++ b/server/upstream/upstream.go
@@ -83,7 +83,7 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() {
cachedResponse = cachedValue.(gitea.FileResponse)
} else {
- res, err = giteaClient.ServeRawContent(uri)
+ res, err = giteaClient.ServeRawContent(o.generateUriClientArgs())
}
log.Debug().Msg("Aquisting")