From f624f8a090187126492957eec55c52c51c0a2974 Mon Sep 17 00:00:00 2001 From: Neha Sherpa Date: Tue, 3 Feb 2026 17:13:39 -0800 Subject: [PATCH] feat: Add GitHub App authentication for git cloning and releases Implements GitHub App authentication to enable authenticated access to private repositories and releases. This provides better security and higher rate limits compared to personal access tokens. Key features: - JWT generation with RSA signing for GitHub App authentication - Installation token management with 1-hour expiration and auto-refresh - Token caching to minimize GitHub API calls - Credential injection into git clone/fetch operations - Support for both git strategy and github-releases strategy - Falls back to system credentials if GitHub App not configured Implementation details: - New internal/githubapp package for authentication logic - Updated gitclone package to support credential providers - Modified git commands to inject tokens into GitHub URLs - Added GitHub App configuration blocks to both strategies - Updated configuration examples in cachew.hcl Usage: Configure GitHub App credentials (app ID, private key path, and installation IDs per org) in the git and github-releases strategy blocks to enable authenticated access. --- cachew.hcl | 10 +- cmd/cachewd/main.go | 42 +++++-- docker/Dockerfile | 2 +- go.mod | 1 + go.sum | 2 + internal/config/kong.go | 16 +++ internal/gitclone/command.go | 42 ++++++- internal/gitclone/command_test.go | 4 +- internal/gitclone/manager.go | 66 ++++++---- internal/gitclone/manager_test.go | 12 +- internal/githubapp/config.go | 70 +++++++++++ internal/githubapp/jwt.go | 78 ++++++++++++ internal/githubapp/tokens.go | 176 +++++++++++++++++++++++++++ internal/strategy/git/git.go | 30 ++++- internal/strategy/github_releases.go | 54 +++++--- 15 files changed, 536 insertions(+), 69 deletions(-) create mode 100644 internal/githubapp/config.go create mode 100644 internal/githubapp/jwt.go create mode 100644 internal/githubapp/tokens.go diff --git a/cachew.hcl b/cachew.hcl index 6cfa596..5508537 100644 --- a/cachew.hcl +++ b/cachew.hcl @@ -1,7 +1,3 @@ -# strategy git {} -# strategy docker {} -# strategy hermit {} - # Artifactory caching proxy strategy # artifactory "example.jfrog.io" { # target = "https://example.jfrog.io" @@ -12,6 +8,12 @@ log { level = "debug" } +github-app { + app-id = "${GITHUB_APP_ID}" + private-key-path = "${GITHUB_APP_PRIVATE_KEY_PATH}" + installations-json = "${GITHUB_APP_INSTALLATIONS}" +} + git { mirror-root = "./state/git-mirrors" bundle-interval = "24h" diff --git a/cmd/cachewd/main.go b/cmd/cachewd/main.go index be34db2..4478938 100644 --- a/cmd/cachewd/main.go +++ b/cmd/cachewd/main.go @@ -19,6 +19,7 @@ import ( "github.com/block/cachew/internal/cache" "github.com/block/cachew/internal/config" + "github.com/block/cachew/internal/githubapp" "github.com/block/cachew/internal/httputil" "github.com/block/cachew/internal/jobscheduler" "github.com/block/cachew/internal/logging" @@ -34,6 +35,7 @@ type GlobalConfig struct { SchedulerConfig jobscheduler.Config `embed:"" hcl:"scheduler,block" prefix:"scheduler-"` LoggingConfig logging.Config `embed:"" hcl:"log,block" prefix:"log-"` MetricsConfig metrics.Config `embed:"" hcl:"metrics,block" prefix:"metrics-"` + GithubAppConfig githubapp.Config `embed:"" hcl:"github-app,block" prefix:"github-app-"` } var cli struct { @@ -62,6 +64,8 @@ func main() { scheduler := jobscheduler.New(ctx, cli.SchedulerConfig) + githubapp.SetShared(initializeGitHubApp(ctx, logger, cli.GithubAppConfig)) + cr, sr := newRegistries(scheduler) // Commands @@ -127,6 +131,32 @@ func printSchema(kctx *kong.Context, cr *cache.Registry, sr *strategy.Registry) } } +func initializeGitHubApp(ctx context.Context, logger *slog.Logger, cfg githubapp.Config) *githubapp.TokenManager { + if err := cfg.Initialize(logger); err != nil { + logger.WarnContext(ctx, "Failed to initialize GitHub App config", + slog.String("error", err.Error())) + return nil + } + + if !cfg.IsConfigured() { + return nil + } + + tm, err := githubapp.NewTokenManager(cfg, githubapp.DefaultTokenCacheConfig()) + if err != nil { + logger.ErrorContext(ctx, "Failed to create GitHub App token manager", + slog.String("error", err.Error())) + return nil + } + + logger.InfoContext(ctx, "Global GitHub App authentication enabled", + slog.String("app_id", cfg.AppID), + slog.String("private_key_path", cfg.PrivateKeyPath), + slog.Int("installations", len(cfg.Installations))) + + return tm +} + func newMux(ctx context.Context, cr *cache.Registry, sr *strategy.Registry, providersConfig *hcl.AST) (*http.ServeMux, error) { mux := http.NewServeMux() @@ -140,7 +170,7 @@ func newMux(ctx context.Context, cr *cache.Registry, sr *strategy.Registry, prov _, _ = w.Write([]byte("OK")) //nolint:errcheck }) - if err := config.Load(ctx, cr, sr, providersConfig, mux, parseEnvars()); err != nil { + if err := config.Load(ctx, cr, sr, providersConfig, mux, config.ParseEnvars()); err != nil { return nil, fmt.Errorf("load config: %w", err) } @@ -171,13 +201,3 @@ func newServer(ctx context.Context, logger *slog.Logger, mux *http.ServeMux) *ht }, } } - -func parseEnvars() map[string]string { - envars := map[string]string{} - for _, env := range os.Environ() { - if key, value, ok := strings.Cut(env, "="); ok { - envars[key] = value - } - } - return envars -} diff --git a/docker/Dockerfile b/docker/Dockerfile index b7bcb2b..90ce866 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,7 +6,7 @@ ARG TARGETARCH SHELL ["/bin/sh", "-o", "pipefail", "-c"] # Install runtime dependencies for git operations and TLS -RUN apk add --no-cache ca-certificates git tzdata && \ +RUN apk add --no-cache ca-certificates git git-daemon tzdata && \ addgroup -g 1000 cachew && \ adduser -D -u 1000 -G cachew cachew diff --git a/go.mod b/go.mod index e1a978a..6619cf1 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.25.5 require ( github.com/alecthomas/hcl/v2 v2.5.0 github.com/alecthomas/kong v1.13.0 + github.com/golang-jwt/jwt/v5 v5.3.1 github.com/goproxy/goproxy v0.25.0 github.com/lmittmann/tint v1.1.2 github.com/minio/minio-go/v7 v7.0.97 diff --git a/go.sum b/go.sum index 4fb62b8..2672c3d 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,8 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= diff --git a/internal/config/kong.go b/internal/config/kong.go index e529b39..82cf6ec 100644 --- a/internal/config/kong.go +++ b/internal/config/kong.go @@ -3,6 +3,7 @@ package config import ( "fmt" "io" + "os" "strings" "github.com/alecthomas/hcl/v2" @@ -14,9 +15,24 @@ func KongLoader[GlobalConfig any](r io.Reader) (kong.Resolver, error) { if err != nil { return nil, fmt.Errorf("failed to parse HCL: %w", err) } + + // Expand environment variables in the AST + expandVars(ast, ParseEnvars()) + return &kongResolver{flattenHCL(ast)}, nil } +// ParseEnvars returns a map of all environment variables. +func ParseEnvars() map[string]string { + envars := map[string]string{} + for _, env := range os.Environ() { + if key, value, ok := strings.Cut(env, "="); ok { + envars[key] = value + } + } + return envars +} + type kongResolver struct { values map[string]any } diff --git a/internal/gitclone/command.go b/internal/gitclone/command.go index 31a3a17..5aceda4 100644 --- a/internal/gitclone/command.go +++ b/internal/gitclone/command.go @@ -5,13 +5,23 @@ package gitclone import ( "bufio" "context" + "fmt" "os/exec" "strings" "github.com/alecthomas/errors" ) -func gitCommand(ctx context.Context, url string, args ...string) (*exec.Cmd, error) { +func gitCommand(ctx context.Context, url string, credentialProvider CredentialProvider, args ...string) (*exec.Cmd, error) { + modifiedURL := url + if credentialProvider != nil && strings.Contains(url, "github.com") { + token, err := credentialProvider.GetTokenForURL(ctx, url) + if err == nil && token != "" { + modifiedURL = injectTokenIntoURL(url, token) + } + // If error getting token, fall back to original URL (system credentials) + } + configArgs, err := getInsteadOfDisableArgsForURL(ctx, url) if err != nil { return nil, errors.Wrap(err, "get insteadOf disable args") @@ -23,8 +33,34 @@ func gitCommand(ctx context.Context, url string, args ...string) (*exec.Cmd, err } allArgs = append(allArgs, args...) - cmd := exec.CommandContext(ctx, "git", allArgs...) - return cmd, nil + // Replace URL in args if it was modified for authentication + if modifiedURL != url { + for i, arg := range allArgs { + if arg == url { + allArgs[i] = modifiedURL + } + } + } + + return exec.CommandContext(ctx, "git", allArgs...), nil +} + +// Converts https://github.com/org/repo to https://x-access-token:TOKEN@github.com/org/repo +func injectTokenIntoURL(url, token string) string { + if token == "" { + return url + } + + if strings.HasPrefix(url, "https://github.com/") { + return strings.Replace(url, "https://github.com/", fmt.Sprintf("https://x-access-token:%s@github.com/", token), 1) + } + + // Upgrade http to https when adding token for security + if strings.HasPrefix(url, "http://github.com/") { + return strings.Replace(url, "http://github.com/", fmt.Sprintf("https://x-access-token:%s@github.com/", token), 1) + } + + return url } func getInsteadOfDisableArgsForURL(ctx context.Context, targetURL string) ([]string, error) { diff --git a/internal/gitclone/command_test.go b/internal/gitclone/command_test.go index e700a21..12bc2d3 100644 --- a/internal/gitclone/command_test.go +++ b/internal/gitclone/command_test.go @@ -45,7 +45,7 @@ func TestGetInsteadOfDisableArgsForURL(t *testing.T) { func TestGitCommand(t *testing.T) { ctx := context.Background() - cmd, err := gitCommand(ctx, "https://github.com/user/repo", "version") + cmd, err := gitCommand(ctx, "https://github.com/user/repo", nil, "version") assert.NoError(t, err) assert.NotZero(t, cmd) @@ -59,7 +59,7 @@ func TestGitCommand(t *testing.T) { func TestGitCommandWithEmptyURL(t *testing.T) { ctx := context.Background() - cmd, err := gitCommand(ctx, "", "version") + cmd, err := gitCommand(ctx, "", nil, "version") assert.NoError(t, err) assert.NotZero(t, cmd) diff --git a/internal/gitclone/manager.go b/internal/gitclone/manager.go index f332a2d..ccc5382 100644 --- a/internal/gitclone/manager.go +++ b/internal/gitclone/manager.go @@ -74,24 +74,31 @@ type Config struct { GitConfig GitTuningConfig } +// CredentialProvider provides credentials for git operations. +type CredentialProvider interface { + GetTokenForURL(ctx context.Context, url string) (string, error) +} + type Repository struct { - mu sync.RWMutex - state State - path string - upstreamURL string - lastFetch time.Time - lastRefCheck time.Time - refCheckValid bool - fetchSem chan struct{} + mu sync.RWMutex + state State + path string + upstreamURL string + lastFetch time.Time + lastRefCheck time.Time + refCheckValid bool + fetchSem chan struct{} + credentialProvider CredentialProvider } type Manager struct { - config Config - clones map[string]*Repository - clonesMu sync.RWMutex + config Config + clones map[string]*Repository + clonesMu sync.RWMutex + credentialProvider CredentialProvider } -func NewManager(_ context.Context, config Config) (*Manager, error) { +func NewManager(_ context.Context, config Config, credentialProvider CredentialProvider) (*Manager, error) { if config.RootDir == "" { return nil, errors.New("RootDir is required") } @@ -101,8 +108,9 @@ func NewManager(_ context.Context, config Config) (*Manager, error) { } return &Manager{ - config: config, - clones: make(map[string]*Repository), + config: config, + clones: make(map[string]*Repository), + credentialProvider: credentialProvider, }, nil } @@ -125,10 +133,11 @@ func (m *Manager) GetOrCreate(_ context.Context, upstreamURL string) (*Repositor clonePath := m.clonePathForURL(upstreamURL) repo = &Repository{ - state: StateEmpty, - path: clonePath, - upstreamURL: upstreamURL, - fetchSem: make(chan struct{}, 1), + state: StateEmpty, + path: clonePath, + upstreamURL: upstreamURL, + fetchSem: make(chan struct{}, 1), + credentialProvider: m.credentialProvider, } gitDir := filepath.Join(clonePath, ".git") @@ -152,6 +161,10 @@ func (m *Manager) Config() Config { return m.config } +func (m *Manager) CredentialProvider() CredentialProvider { + return m.credentialProvider +} + func (m *Manager) DiscoverExisting(_ context.Context) ([]*Repository, error) { var discovered []*Repository err := filepath.Walk(m.config.RootDir, func(path string, info os.FileInfo, err error) error { @@ -195,10 +208,11 @@ func (m *Manager) DiscoverExisting(_ context.Context) ([]*Repository, error) { upstreamURL := "https://" + host + "/" + repoPath repo := &Repository{ - state: StateReady, - path: path, - upstreamURL: upstreamURL, - fetchSem: make(chan struct{}, 1), + state: StateReady, + path: path, + upstreamURL: upstreamURL, + fetchSem: make(chan struct{}, 1), + credentialProvider: m.credentialProvider, } repo.fetchSem <- struct{}{} @@ -304,7 +318,7 @@ func (r *Repository) executeClone(ctx context.Context, config Config) error { r.upstreamURL, r.path, } - cmd, err := gitCommand(ctx, r.upstreamURL, args...) + cmd, err := gitCommand(ctx, r.upstreamURL, r.credentialProvider, args...) if err != nil { return errors.Wrap(err, "create git command") } @@ -320,7 +334,7 @@ func (r *Repository) executeClone(ctx context.Context, config Config) error { return errors.Wrapf(err, "configure fetch refspec: %s", string(output)) } - cmd, err = gitCommand(ctx, r.upstreamURL, "-C", r.path, + cmd, err = gitCommand(ctx, r.upstreamURL, r.credentialProvider, "-C", r.path, "-c", "http.postBuffer="+strconv.Itoa(config.GitConfig.PostBuffer), "-c", "http.lowSpeedLimit="+strconv.Itoa(config.GitConfig.LowSpeedLimit), "-c", "http.lowSpeedTime="+strconv.Itoa(int(config.GitConfig.LowSpeedTime.Seconds())), @@ -357,7 +371,7 @@ func (r *Repository) Fetch(ctx context.Context, config Config) error { r.mu.Lock() // #nosec G204 - r.path is controlled by us - cmd, err := gitCommand(ctx, r.upstreamURL, "-C", r.path, + cmd, err := gitCommand(ctx, r.upstreamURL, r.credentialProvider, "-C", r.path, "-c", "http.postBuffer="+strconv.Itoa(config.GitConfig.PostBuffer), "-c", "http.lowSpeedLimit="+strconv.Itoa(config.GitConfig.LowSpeedLimit), "-c", "http.lowSpeedTime="+strconv.Itoa(int(config.GitConfig.LowSpeedTime.Seconds())), @@ -447,7 +461,7 @@ func (r *Repository) GetLocalRefs(ctx context.Context) (map[string]string, error func (r *Repository) GetUpstreamRefs(ctx context.Context) (map[string]string, error) { // #nosec G204 - r.upstreamURL is controlled by us - cmd, err := gitCommand(ctx, r.upstreamURL, "ls-remote", r.upstreamURL) + cmd, err := gitCommand(ctx, r.upstreamURL, r.credentialProvider, "ls-remote", r.upstreamURL) if err != nil { return nil, errors.Wrap(err, "create git command") } diff --git a/internal/gitclone/manager_test.go b/internal/gitclone/manager_test.go index f538ae9..b115daa 100644 --- a/internal/gitclone/manager_test.go +++ b/internal/gitclone/manager_test.go @@ -21,7 +21,7 @@ func TestNewManager(t *testing.T) { GitConfig: DefaultGitTuningConfig(), } - manager, err := NewManager(context.Background(), config) + manager, err := NewManager(context.Background(), config, nil) assert.NoError(t, err) assert.NotZero(t, manager) assert.Equal(t, tmpDir, manager.config.RootDir) @@ -33,7 +33,7 @@ func TestNewManager_RequiresRootDir(t *testing.T) { RefCheckInterval: 10 * time.Second, } - _, err := NewManager(context.Background(), config) + _, err := NewManager(context.Background(), config, nil) assert.Error(t, err) assert.Contains(t, err.Error(), "RootDir is required") } @@ -47,7 +47,7 @@ func TestManager_GetOrCreate(t *testing.T) { GitConfig: DefaultGitTuningConfig(), } - manager, err := NewManager(context.Background(), config) + manager, err := NewManager(context.Background(), config, nil) assert.NoError(t, err) upstreamURL := "https://github.com/user/repo" @@ -73,7 +73,7 @@ func TestManager_GetOrCreate_ExistingClone(t *testing.T) { GitConfig: DefaultGitTuningConfig(), } - manager, err := NewManager(context.Background(), config) + manager, err := NewManager(context.Background(), config, nil) assert.NoError(t, err) repoPath := filepath.Join(tmpDir, "github.com", "user", "repo") @@ -98,7 +98,7 @@ func TestManager_Get(t *testing.T) { GitConfig: DefaultGitTuningConfig(), } - manager, err := NewManager(context.Background(), config) + manager, err := NewManager(context.Background(), config, nil) assert.NoError(t, err) upstreamURL := "https://github.com/user/repo" @@ -123,7 +123,7 @@ func TestManager_DiscoverExisting(t *testing.T) { GitConfig: DefaultGitTuningConfig(), } - manager, err := NewManager(context.Background(), config) + manager, err := NewManager(context.Background(), config, nil) assert.NoError(t, err) repos := []string{ diff --git a/internal/githubapp/config.go b/internal/githubapp/config.go new file mode 100644 index 0000000..b9927af --- /dev/null +++ b/internal/githubapp/config.go @@ -0,0 +1,70 @@ +// Package githubapp provides GitHub App authentication and token management. +package githubapp + +import ( + "encoding/json" + "log/slog" + "time" + + "github.com/alecthomas/errors" +) + +type Config struct { + AppID string `hcl:"app-id" help:"GitHub App ID"` + PrivateKeyPath string `hcl:"private-key-path" help:"Path to GitHub App private key (PEM format)"` + InstallationsJSON string `hcl:"installations-json" help:"JSON string mapping org names to installation IDs"` + + // Populated from InstallationsJSON during Initialize(). Not exposed in HCL. + Installations map[string]string +} + +type TokenCacheConfig struct { + RefreshBuffer time.Duration // How early to refresh before expiration + JWTExpiration time.Duration // GitHub allows max 10 minutes +} + +func DefaultTokenCacheConfig() TokenCacheConfig { + return TokenCacheConfig{ + RefreshBuffer: 5 * time.Minute, + JWTExpiration: 10 * time.Minute, + } +} + +// Initialize must be called after loading config to parse InstallationsJSON into Installations map. +func (c *Config) Initialize(logger *slog.Logger) error { + if c.InstallationsJSON == "" { + return errors.New("installations-json is required") + } + + var installations map[string]string + if err := json.Unmarshal([]byte(c.InstallationsJSON), &installations); err != nil { + logger.Error("Failed to parse installations-json", + "error", err, + "installations_json", c.InstallationsJSON) + return errors.Wrap(err, "parse installations-json") + } + + if len(installations) == 0 { + return errors.New("installations-json must contain at least one organization") + } + + c.Installations = installations + + logger.Info("GitHub App config initialized", + "app_id", c.AppID, + "private_key_path", c.PrivateKeyPath, + "installations_json", c.InstallationsJSON) + + return nil +} + +func (c *Config) IsConfigured() bool { + return c.AppID != "" && c.PrivateKeyPath != "" && len(c.Installations) > 0 +} + +func (c *Config) GetInstallationID(org string) string { + if c.Installations == nil { + return "" + } + return c.Installations[org] +} diff --git a/internal/githubapp/jwt.go b/internal/githubapp/jwt.go new file mode 100644 index 0000000..a2b5edb --- /dev/null +++ b/internal/githubapp/jwt.go @@ -0,0 +1,78 @@ +package githubapp + +import ( + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "os" + "time" + + "github.com/alecthomas/errors" + "github.com/golang-jwt/jwt/v5" +) + +type JWTGenerator struct { + appID string + privateKey *rsa.PrivateKey + expiration time.Duration +} + +func NewJWTGenerator(appID, privateKeyPath string, expiration time.Duration) (*JWTGenerator, error) { + privateKey, err := loadPrivateKey(privateKeyPath) + if err != nil { + return nil, errors.Wrap(err, "load private key") + } + + return &JWTGenerator{ + appID: appID, + privateKey: privateKey, + expiration: expiration, + }, nil +} + +func (g *JWTGenerator) GenerateJWT() (string, error) { + now := time.Now() + + claims := jwt.RegisteredClaims{ + IssuedAt: jwt.NewNumericDate(now), + ExpiresAt: jwt.NewNumericDate(now.Add(g.expiration)), + Issuer: g.appID, + } + + token := jwt.NewWithClaims(jwt.SigningMethodRS256, claims) + signedToken, err := token.SignedString(g.privateKey) + if err != nil { + return "", errors.Wrap(err, "sign JWT") + } + + return signedToken, nil +} + +func loadPrivateKey(path string) (*rsa.PrivateKey, error) { + keyData, err := os.ReadFile(path) + if err != nil { + return nil, errors.Wrapf(err, "read private key file: %s", path) + } + + block, _ := pem.Decode(keyData) + if block == nil { + return nil, errors.Errorf("failed to decode PEM block from private key file: %s", path) + } + + privateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes) + if err == nil { + return privateKey, nil + } + + key, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return nil, errors.Wrap(err, "parse private key (tried both PKCS1 and PKCS8)") + } + + rsaKey, ok := key.(*rsa.PrivateKey) + if !ok { + return nil, errors.Errorf("private key is not RSA (type: %T)", key) + } + + return rsaKey, nil +} diff --git a/internal/githubapp/tokens.go b/internal/githubapp/tokens.go new file mode 100644 index 0000000..115ea38 --- /dev/null +++ b/internal/githubapp/tokens.go @@ -0,0 +1,176 @@ +package githubapp + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "strings" + "sync" + "time" + + "github.com/alecthomas/errors" + + "github.com/block/cachew/internal/logging" +) + +var ( + sharedTokenManager *TokenManager + sharedTokenManagerMu sync.RWMutex +) + +func SetShared(tm *TokenManager) { + sharedTokenManagerMu.Lock() + defer sharedTokenManagerMu.Unlock() + sharedTokenManager = tm +} + +func GetShared() *TokenManager { + sharedTokenManagerMu.RLock() + defer sharedTokenManagerMu.RUnlock() + return sharedTokenManager +} + +type TokenManager struct { + config Config + cacheConfig TokenCacheConfig + jwtGenerator *JWTGenerator + httpClient *http.Client + + mu sync.RWMutex + tokens map[string]*cachedToken +} + +type cachedToken struct { + token string + expiresAt time.Time +} + +func NewTokenManager(config Config, cacheConfig TokenCacheConfig) (*TokenManager, error) { + if !config.IsConfigured() { + return nil, errors.New("GitHub App not configured") + } + + jwtGenerator, err := NewJWTGenerator(config.AppID, config.PrivateKeyPath, cacheConfig.JWTExpiration) + if err != nil { + return nil, errors.Wrap(err, "create JWT generator") + } + + return &TokenManager{ + config: config, + cacheConfig: cacheConfig, + jwtGenerator: jwtGenerator, + httpClient: http.DefaultClient, + tokens: make(map[string]*cachedToken), + }, nil +} + +func (tm *TokenManager) GetTokenForOrg(ctx context.Context, org string) (string, error) { + if tm == nil { + return "", errors.New("token manager not initialized") + } + logger := logging.FromContext(ctx).With(slog.String("org", org)) + + installationID := tm.config.GetInstallationID(org) + if installationID == "" { + return "", errors.Errorf("no GitHub App installation configured for org: %s", org) + } + + tm.mu.RLock() + cached, exists := tm.tokens[org] + tm.mu.RUnlock() + + if exists && time.Now().Add(tm.cacheConfig.RefreshBuffer).Before(cached.expiresAt) { + logger.DebugContext(ctx, "Using cached GitHub App token") + return cached.token, nil + } + + logger.DebugContext(ctx, "Fetching new GitHub App installation token", + slog.String("installation_id", installationID)) + + token, expiresAt, err := tm.fetchInstallationToken(ctx, installationID) + if err != nil { + return "", errors.Wrap(err, "fetch installation token") + } + + tm.mu.Lock() + tm.tokens[org] = &cachedToken{ + token: token, + expiresAt: expiresAt, + } + tm.mu.Unlock() + + logger.InfoContext(ctx, "GitHub App token refreshed", + slog.Time("expires_at", expiresAt)) + + return token, nil +} + +func (tm *TokenManager) GetTokenForURL(ctx context.Context, url string) (string, error) { + if tm == nil { + return "", errors.New("token manager not initialized") + } + org, err := extractOrgFromURL(url) + if err != nil { + return "", err + } + + return tm.GetTokenForOrg(ctx, org) +} + +func (tm *TokenManager) fetchInstallationToken(ctx context.Context, installationID string) (string, time.Time, error) { + jwt, err := tm.jwtGenerator.GenerateJWT() + if err != nil { + return "", time.Time{}, errors.Wrap(err, "generate JWT") + } + + url := fmt.Sprintf("https://api.github.com/app/installations/%s/access_tokens", installationID) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, nil) + if err != nil { + return "", time.Time{}, errors.Wrap(err, "create request") + } + + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("Authorization", "Bearer "+jwt) + req.Header.Set("X-Github-Api-Version", "2022-11-28") + + resp, err := tm.httpClient.Do(req) + if err != nil { + return "", time.Time{}, errors.Wrap(err, "execute request") + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + return "", time.Time{}, errors.Errorf("GitHub API returned status %d", resp.StatusCode) + } + + var result struct { + Token string `json:"token"` + ExpiresAt time.Time `json:"expires_at"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return "", time.Time{}, errors.Wrap(err, "decode response") + } + + return result.Token, result.ExpiresAt, nil +} + +func extractOrgFromURL(url string) (string, error) { + url = strings.TrimPrefix(url, "https://") + url = strings.TrimPrefix(url, "http://") + url = strings.TrimPrefix(url, "git@") + + if !strings.HasPrefix(url, "github.com/") && !strings.HasPrefix(url, "github.com:") { + return "", errors.Errorf("not a GitHub URL: %s", url) + } + url = strings.TrimPrefix(url, "github.com/") + url = strings.TrimPrefix(url, "github.com:") + + parts := strings.Split(url, "/") + if len(parts) < 1 || parts[0] == "" { + return "", errors.Errorf("cannot extract org from URL: %s", url) + } + + return parts[0], nil +} diff --git a/internal/strategy/git/git.go b/internal/strategy/git/git.go index 8c2fb6e..56e9474 100644 --- a/internal/strategy/git/git.go +++ b/internal/strategy/git/git.go @@ -21,6 +21,7 @@ import ( "github.com/block/cachew/internal/cache" "github.com/block/cachew/internal/gitclone" + "github.com/block/cachew/internal/githubapp" "github.com/block/cachew/internal/jobscheduler" "github.com/block/cachew/internal/logging" "github.com/block/cachew/internal/strategy" @@ -38,6 +39,7 @@ type Config struct { RefCheckInterval time.Duration `hcl:"ref-check-interval,optional" help:"How long to cache ref checks." default:"10s"` BundleInterval time.Duration `hcl:"bundle-interval,optional" help:"How often to generate bundles. 0 disables bundling." default:"0"` SnapshotInterval time.Duration `hcl:"snapshot-interval,optional" help:"How often to generate tar.zstd snapshots. 0 disables snapshots." default:"0"` + CloneDepth int `hcl:"clone-depth,optional" help:"Depth for shallow clones. 0 means full clone." default:"0"` } type Strategy struct { @@ -50,6 +52,7 @@ type Strategy struct { scheduler jobscheduler.Scheduler spoolsMu sync.Mutex spools map[string]*RepoSpools + tokenManager *githubapp.TokenManager } func New(ctx context.Context, config Config, scheduler jobscheduler.Scheduler, cache cache.Cache, mux strategy.Mux) (*Strategy, error) { @@ -67,12 +70,20 @@ func New(ctx context.Context, config Config, scheduler jobscheduler.Scheduler, c config.RefCheckInterval = 10 * time.Second } + // Use shared GitHub App token manager if configured + tokenManager := githubapp.GetShared() + if tokenManager != nil { + logger.InfoContext(ctx, "Using global GitHub App authentication for git strategy") + } else { + logger.WarnContext(ctx, "GitHub App not configured, using system git credentials") + } + cloneManager, err := gitclone.NewManager(ctx, gitclone.Config{ RootDir: config.MirrorRoot, FetchInterval: config.FetchInterval, RefCheckInterval: config.RefCheckInterval, GitConfig: gitclone.DefaultGitTuningConfig(), - }) + }, tokenManager) if err != nil { return nil, errors.Wrap(err, "create clone manager") } @@ -91,6 +102,7 @@ func New(ctx context.Context, config Config, scheduler jobscheduler.Scheduler, c ctx: ctx, scheduler: scheduler.WithQueuePrefix("git"), spools: make(map[string]*RepoSpools), + tokenManager: tokenManager, } existing, err := s.cloneManager.DiscoverExisting(ctx) @@ -113,6 +125,22 @@ func New(ctx context.Context, config Config, scheduler jobscheduler.Scheduler, c req.URL.Host = req.PathValue("host") req.URL.Path = "/" + req.PathValue("path") req.Host = req.URL.Host + + // Inject GitHub App authentication for github.com requests + if s.tokenManager != nil && req.URL.Host == "github.com" { + // Extract org from path (e.g., /squareup/blox.git/...) + parts := strings.Split(strings.TrimPrefix(req.URL.Path, "/"), "/") + if len(parts) >= 1 && parts[0] != "" { + org := parts[0] + token, err := s.tokenManager.GetTokenForOrg(req.Context(), org) + if err == nil && token != "" { + // Inject token as Basic auth with "x-access-token" username + req.SetBasicAuth("x-access-token", token) + logger.DebugContext(req.Context(), "Injecting GitHub App auth into upstream request", + slog.String("org", org)) + } + } + } }, Transport: s.httpClient.Transport, ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) { diff --git a/internal/strategy/github_releases.go b/internal/strategy/github_releases.go index fa2da49..47f1419 100644 --- a/internal/strategy/github_releases.go +++ b/internal/strategy/github_releases.go @@ -11,6 +11,7 @@ import ( "github.com/alecthomas/errors" "github.com/block/cachew/internal/cache" + "github.com/block/cachew/internal/githubapp" "github.com/block/cachew/internal/httputil" "github.com/block/cachew/internal/logging" "github.com/block/cachew/internal/strategy/handler" @@ -21,27 +22,35 @@ func RegisterGitHubReleases(r *Registry) { } type GitHubReleasesConfig struct { - Token string `hcl:"token" help:"GitHub token for authentication."` + Token string `hcl:"token,optional" help:"GitHub token for authentication."` PrivateOrgs []string `hcl:"private-orgs" help:"List of private GitHub organisations."` } // The GitHubReleases strategy fetches private (and public) release binaries from GitHub. type GitHubReleases struct { - config GitHubReleasesConfig - cache cache.Cache - client *http.Client + config GitHubReleasesConfig + cache cache.Cache + client *http.Client + tokenManager *githubapp.TokenManager } // NewGitHubReleases creates a [Strategy] that fetches private (and public) release binaries from GitHub. func NewGitHubReleases(ctx context.Context, config GitHubReleasesConfig, cache cache.Cache, mux Mux) (*GitHubReleases, error) { - s := &GitHubReleases{ - config: config, - cache: cache, - client: http.DefaultClient, - } logger := logging.FromContext(ctx) - if config.Token == "" { - logger.WarnContext(ctx, "No token configured for github-releases strategy") + + // Use shared GitHub App token manager if configured + tokenManager := githubapp.GetShared() + if tokenManager != nil { + logger.InfoContext(ctx, "Using global GitHub App authentication for private github-releases strategy") + } else if config.Token == "" { + logger.WarnContext(ctx, "No authentication configured for github-releases strategy") + } + + s := &GitHubReleases{ + config: config, + cache: cache, + client: http.DefaultClient, + tokenManager: tokenManager, } // eg. https://github.com/alecthomas/chroma/releases/download/v2.21.1/chroma-2.21.1-darwin-amd64.tar.gz h := handler.New(s.client, cache). @@ -68,16 +77,31 @@ var _ Strategy = (*GitHubReleases)(nil) func (g *GitHubReleases) String() string { return "github-releases" } // newGitHubRequest creates a new HTTP request with GitHub API headers and authentication. -func (g *GitHubReleases) newGitHubRequest(ctx context.Context, url, accept string) (*http.Request, error) { +func (g *GitHubReleases) newGitHubRequest(ctx context.Context, url, accept, org string) (*http.Request, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return nil, errors.Wrap(err, "create request") } req.Header.Set("Accept", accept) req.Header.Set("X-Github-Api-Version", "2022-11-28") - if g.config.Token != "" { + + // Try GitHub App authentication first, fall back to static token + if g.tokenManager != nil && org != "" { + token, err := g.tokenManager.GetTokenForOrg(ctx, org) + if err != nil { + logging.FromContext(ctx).WarnContext(ctx, "Failed to get GitHub App token, falling back to static token", + slog.String("org", org), + slog.String("error", err.Error())) + } else if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + } + + // Fall back to static token if GitHub App not used or failed + if req.Header.Get("Authorization") == "" && g.config.Token != "" { req.Header.Set("Authorization", "Bearer "+g.config.Token) } + return req, nil } @@ -107,7 +131,7 @@ func (g *GitHubReleases) downloadRelease(ctx context.Context, org, repo, release // Use GitHub API to get release info and find the asset logger.DebugContext(ctx, "Using GitHub API for private release") apiURL := fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/tags/%s", org, repo, release) - req, err := g.newGitHubRequest(ctx, apiURL, "application/vnd.github+json") + req, err := g.newGitHubRequest(ctx, apiURL, "application/vnd.github+json", org) if err != nil { return nil, httputil.Errorf(http.StatusInternalServerError, "create API request") } @@ -148,7 +172,7 @@ func (g *GitHubReleases) downloadRelease(ctx context.Context, org, repo, release logger.DebugContext(ctx, "Found asset in release", slog.String("asset_url", assetURL)) // Create request for the asset download - req, err = g.newGitHubRequest(ctx, assetURL, "application/octet-stream") + req, err = g.newGitHubRequest(ctx, assetURL, "application/octet-stream", org) if err != nil { return nil, httputil.Errorf(http.StatusInternalServerError, "create asset request failed: %w", err) }