From f5d0dc7447c5dd5a079b518a457d7559015a3fce Mon Sep 17 00:00:00 2001
From: 6543 <6543@noreply.codeberg.org>
Date: Sun, 27 Mar 2022 21:54:06 +0200
Subject: [PATCH] Add pipeline (#65)
close #54
Co-authored-by: 6543 <6543@obermui.de>
Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/65
Reviewed-by: Andreas Shimokawa <ashimokawa@noreply.codeberg.org>
---
.gitignore | 1 +
.woodpecker.yml | 25 +++++++++++++++++++++++++
cmd/certs.go | 5 +++--
cmd/main.go | 8 ++++----
main.go | 6 ++----
server/certificates/acme_account.go | 2 ++
server/certificates/certificates.go | 13 +++++++++----
server/database/setup.go | 16 ++--------------
server/handler.go | 7 ++++---
server/handler_test.go | 3 ++-
server/try.go | 4 ++--
server/upstream/helper.go | 4 ++--
12 files changed, 58 insertions(+), 36 deletions(-)
create mode 100644 .woodpecker.yml
diff --git a/.gitignore b/.gitignore
index 3117f77..dfe69ac 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,3 +4,4 @@ key-database.pogreb/
acme-account.json
build/
vendor/
+pages
diff --git a/.woodpecker.yml b/.woodpecker.yml
new file mode 100644
index 0000000..eafb569
--- /dev/null
+++ b/.woodpecker.yml
@@ -0,0 +1,25 @@
+pipeline:
+ # use vendor to cache dependencies
+ vendor:
+ image: golang
+ commands:
+ - go mod vendor
+
+ lint:
+ image: golangci/golangci-lint:v1.45.2
+ commands:
+ - go version
+ - go install mvdan.cc/gofumpt@latest
+ - "[ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }"
+ - golangci-lint run
+
+ # # TODO: make tests work
+ # test:
+ # image: golang
+ # commands:
+ # - go test ./...
+
+ build:
+ image: golang
+ commands:
+ - go build
diff --git a/cmd/certs.go b/cmd/certs.go
index 071234b..83f2ac5 100644
--- a/cmd/certs.go
+++ b/cmd/certs.go
@@ -2,6 +2,7 @@ package cmd
import (
"fmt"
+
"github.com/akrylysov/pogreb"
"github.com/urfave/cli/v2"
@@ -12,12 +13,12 @@ var Certs = &cli.Command{
Name: "certs",
Usage: "manage certs manually",
Subcommands: []*cli.Command{
- &cli.Command{
+ {
Name: "list",
Usage: "list all certificates in the database",
Action: listCerts,
},
- &cli.Command{
+ {
Name: "remove",
Usage: "remove a certificate from the database",
Action: removeCert,
diff --git a/cmd/main.go b/cmd/main.go
index fb0c26e..6836cee 100644
--- a/cmd/main.go
+++ b/cmd/main.go
@@ -72,14 +72,14 @@ func Serve(ctx *cli.Context) error {
keyCache := cache.NewKeyValueCache()
challengeCache := cache.NewKeyValueCache()
// canonicalDomainCache stores canonical domains
- var canonicalDomainCache = cache.NewKeyValueCache()
+ canonicalDomainCache := cache.NewKeyValueCache()
// dnsLookupCache stores DNS lookups for custom domains
- var dnsLookupCache = cache.NewKeyValueCache()
+ dnsLookupCache := cache.NewKeyValueCache()
// branchTimestampCache stores branch timestamps for faster cache checking
- var branchTimestampCache = cache.NewKeyValueCache()
+ branchTimestampCache := cache.NewKeyValueCache()
// fileResponseCache stores responses from the Gitea server
// TODO: make this an MRU cache with a size limit
- var fileResponseCache = cache.NewKeyValueCache()
+ fileResponseCache := cache.NewKeyValueCache()
// Create handler based on settings
handler := server.Handler(mainDomainSuffix, []byte(rawDomain),
diff --git a/main.go b/main.go
index 41aba22..205ec93 100644
--- a/main.go
+++ b/main.go
@@ -9,10 +9,8 @@ import (
"codeberg.org/codeberg/pages/cmd"
)
-var (
- // can be changed with -X on compile
- version = "dev"
-)
+// can be changed with -X on compile
+var version = "dev"
func main() {
app := cli.NewApp()
diff --git a/server/certificates/acme_account.go b/server/certificates/acme_account.go
index 2ee2e80..57f4d44 100644
--- a/server/certificates/acme_account.go
+++ b/server/certificates/acme_account.go
@@ -19,9 +19,11 @@ var _ registration.User = &AcmeAccount{}
func (u *AcmeAccount) GetEmail() string {
return u.Email
}
+
func (u AcmeAccount) GetRegistration() *registration.Resource {
return u.Registration
}
+
func (u *AcmeAccount) GetPrivateKey() crypto.PrivateKey {
return u.Key
}
diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go
index 505212f..a13215c 100644
--- a/server/certificates/certificates.go
+++ b/server/certificates/certificates.go
@@ -40,7 +40,8 @@ func TLSConfig(mainDomainSuffix []byte,
giteaRoot, giteaAPIToken, dnsProvider string,
acmeUseRateLimits bool,
keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey,
- certDB database.CertDB) *tls.Config {
+ certDB database.CertDB,
+) *tls.Config {
return &tls.Config{
// check DNS name & get certificate from Let's Encrypt
GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
@@ -146,8 +147,10 @@ func checkUserLimit(user string) error {
return nil
}
-var acmeClient, mainDomainAcmeClient *lego.Client
-var acmeClientCertificateLimitPerUser = map[string]*equalizer.TokenBucket{}
+var (
+ acmeClient, mainDomainAcmeClient *lego.Client
+ acmeClientCertificateLimitPerUser = map[string]*equalizer.TokenBucket{}
+)
// rate limit is 300 / 3 hours, we want 200 / 2 hours but to refill more often, so that's 25 new domains every 15 minutes
// TODO: when this is used a lot, we probably have to think of a somewhat better solution?
@@ -166,6 +169,7 @@ var _ challenge.Provider = AcmeTLSChallengeProvider{}
func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error {
return a.challengeCache.Set(domain, keyAuth, 1*time.Hour)
}
+
func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error {
a.challengeCache.Remove(domain)
return nil
@@ -181,6 +185,7 @@ var _ challenge.Provider = AcmeHTTPChallengeProvider{}
func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error {
return a.challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour)
}
+
func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
a.challengeCache.Remove(domain + "/" + token)
return nil
@@ -388,7 +393,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
log.Printf("[FAIL] Error during json.Marshal(myAcmeAccount), waiting for manual restart to avoid rate limits: %s", err)
select {}
}
- err = ioutil.WriteFile(configFile, acmeAccountJSON, 0600)
+ err = ioutil.WriteFile(configFile, acmeAccountJSON, 0o600)
if err != nil {
log.Printf("[FAIL] Error during ioutil.WriteFile(\"acme-account.json\"), waiting for manual restart to avoid rate limits: %s", err)
select {}
diff --git a/server/database/setup.go b/server/database/setup.go
index f3cac16..e48b661 100644
--- a/server/database/setup.go
+++ b/server/database/setup.go
@@ -13,6 +13,8 @@ import (
"github.com/rs/zerolog/log"
)
+var _ CertDB = aDB{}
+
type aDB struct {
ctx context.Context
cancel context.CancelFunc
@@ -76,20 +78,6 @@ func (p aDB) sync() {
}
}
-func (p aDB) compact() {
- for {
- err := p.intern.Sync()
- if err != nil {
- log.Err(err).Msg("Syncing cert database failed")
- }
- select {
- case <-p.ctx.Done():
- return
- case <-time.After(p.syncInterval):
- }
- }
-}
-
func New(path string) (CertDB, error) {
if path == "" {
return nil, fmt.Errorf("path not set")
diff --git a/server/handler.go b/server/handler.go
index 1aaf476..60f5809 100644
--- a/server/handler.go
+++ b/server/handler.go
@@ -18,7 +18,8 @@ import (
func Handler(mainDomainSuffix, rawDomain []byte,
giteaRoot, rawInfoPage, giteaAPIToken string,
blacklistedPaths, allowedCorsDomains [][]byte,
- dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey) func(ctx *fasthttp.RequestCtx) {
+ dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
+) func(ctx *fasthttp.RequestCtx) {
return func(ctx *fasthttp.RequestCtx) {
log := log.With().Str("Handler", string(ctx.Request.Header.RequestURI())).Logger()
@@ -72,14 +73,14 @@ func Handler(mainDomainSuffix, rawDomain []byte,
// Prepare request information to Gitea
var targetOwner, targetRepo, targetBranch, targetPath string
- var targetOptions = &upstream.Options{
+ targetOptions := &upstream.Options{
ForbiddenMimeTypes: map[string]struct{}{},
TryIndexPages: true,
}
// tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will
// also disallow search indexing and add a Link header to the canonical URL.
- var tryBranch = func(repo string, branch string, path []string, canonicalLink string) bool {
+ tryBranch := func(repo, branch string, path []string, canonicalLink string) bool {
if repo == "" {
return false
}
diff --git a/server/handler_test.go b/server/handler_test.go
index 0ec9fcd..34aed7f 100644
--- a/server/handler_test.go
+++ b/server/handler_test.go
@@ -2,10 +2,11 @@ package server
import (
"fmt"
- "github.com/valyala/fasthttp"
"testing"
"time"
+ "github.com/valyala/fasthttp"
+
"codeberg.org/codeberg/pages/server/cache"
)
diff --git a/server/try.go b/server/try.go
index 31cd7f4..4eda5b2 100644
--- a/server/try.go
+++ b/server/try.go
@@ -19,8 +19,8 @@ func tryUpstream(ctx *fasthttp.RequestCtx,
targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken string,
- canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey) {
-
+ canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
+) {
// check if a canonical domain exists on a request on MainDomain
if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
canonicalDomain, _ := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)
diff --git a/server/upstream/helper.go b/server/upstream/helper.go
index b5ee77a..f61d746 100644
--- a/server/upstream/helper.go
+++ b/server/upstream/helper.go
@@ -27,7 +27,7 @@ func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string, br
result.Branch = branch
if branch == "" {
// Get default branch
- var body = make([]byte, 0)
+ body := make([]byte, 0)
// TODO: use header for API key?
status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"?access_token="+giteaApiToken, 5*time.Second)
if err != nil || status != 200 {
@@ -37,7 +37,7 @@ func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string, br
result.Branch = fastjson.GetString(body, "default_branch")
}
- var body = make([]byte, 0)
+ body := make([]byte, 0)
status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"/branches/"+branch+"?access_token="+giteaApiToken, 5*time.Second)
if err != nil || status != 200 {
return nil