diff --git a/cachew.hcl b/cachew.hcl index 6cfa596..5508537 100644 --- a/cachew.hcl +++ b/cachew.hcl @@ -1,7 +1,3 @@ -# strategy git {} -# strategy docker {} -# strategy hermit {} - # Artifactory caching proxy strategy # artifactory "example.jfrog.io" { # target = "https://example.jfrog.io" @@ -12,6 +8,12 @@ log { level = "debug" } +github-app { + app-id = "${GITHUB_APP_ID}" + private-key-path = "${GITHUB_APP_PRIVATE_KEY_PATH}" + installations-json = "${GITHUB_APP_INSTALLATIONS}" +} + git { mirror-root = "./state/git-mirrors" bundle-interval = "24h" diff --git a/cmd/cachewd/main.go b/cmd/cachewd/main.go index be34db2..4478938 100644 --- a/cmd/cachewd/main.go +++ b/cmd/cachewd/main.go @@ -19,6 +19,7 @@ import ( "github.com/block/cachew/internal/cache" "github.com/block/cachew/internal/config" + "github.com/block/cachew/internal/githubapp" "github.com/block/cachew/internal/httputil" "github.com/block/cachew/internal/jobscheduler" "github.com/block/cachew/internal/logging" @@ -34,6 +35,7 @@ type GlobalConfig struct { SchedulerConfig jobscheduler.Config `embed:"" hcl:"scheduler,block" prefix:"scheduler-"` LoggingConfig logging.Config `embed:"" hcl:"log,block" prefix:"log-"` MetricsConfig metrics.Config `embed:"" hcl:"metrics,block" prefix:"metrics-"` + GithubAppConfig githubapp.Config `embed:"" hcl:"github-app,block" prefix:"github-app-"` } var cli struct { @@ -62,6 +64,8 @@ func main() { scheduler := jobscheduler.New(ctx, cli.SchedulerConfig) + githubapp.SetShared(initializeGitHubApp(ctx, logger, cli.GithubAppConfig)) + cr, sr := newRegistries(scheduler) // Commands @@ -127,6 +131,32 @@ func printSchema(kctx *kong.Context, cr *cache.Registry, sr *strategy.Registry) } } +func initializeGitHubApp(ctx context.Context, logger *slog.Logger, cfg githubapp.Config) *githubapp.TokenManager { + if err := cfg.Initialize(logger); err != nil { + logger.WarnContext(ctx, "Failed to initialize GitHub App config", + slog.String("error", err.Error())) + return nil + } + + if !cfg.IsConfigured() { + return nil + } + + tm, err := githubapp.NewTokenManager(cfg, githubapp.DefaultTokenCacheConfig()) + if err != nil { + logger.ErrorContext(ctx, "Failed to create GitHub App token manager", + slog.String("error", err.Error())) + return nil + } + + logger.InfoContext(ctx, "Global GitHub App authentication enabled", + slog.String("app_id", cfg.AppID), + slog.String("private_key_path", cfg.PrivateKeyPath), + slog.Int("installations", len(cfg.Installations))) + + return tm +} + func newMux(ctx context.Context, cr *cache.Registry, sr *strategy.Registry, providersConfig *hcl.AST) (*http.ServeMux, error) { mux := http.NewServeMux() @@ -140,7 +170,7 @@ func newMux(ctx context.Context, cr *cache.Registry, sr *strategy.Registry, prov _, _ = w.Write([]byte("OK")) //nolint:errcheck }) - if err := config.Load(ctx, cr, sr, providersConfig, mux, parseEnvars()); err != nil { + if err := config.Load(ctx, cr, sr, providersConfig, mux, config.ParseEnvars()); err != nil { return nil, fmt.Errorf("load config: %w", err) } @@ -171,13 +201,3 @@ func newServer(ctx context.Context, logger *slog.Logger, mux *http.ServeMux) *ht }, } } - -func parseEnvars() map[string]string { - envars := map[string]string{} - for _, env := range os.Environ() { - if key, value, ok := strings.Cut(env, "="); ok { - envars[key] = value - } - } - return envars -} diff --git a/docker/Dockerfile b/docker/Dockerfile index b7bcb2b..90ce866 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,7 +6,7 @@ ARG TARGETARCH SHELL ["/bin/sh", "-o", "pipefail", "-c"] # Install runtime dependencies for git operations and TLS -RUN apk add --no-cache ca-certificates git tzdata && \ +RUN apk add --no-cache ca-certificates git git-daemon tzdata && \ addgroup -g 1000 cachew && \ adduser -D -u 1000 -G cachew cachew diff --git a/go.mod b/go.mod index e1a978a..6619cf1 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.25.5 require ( github.com/alecthomas/hcl/v2 v2.5.0 github.com/alecthomas/kong v1.13.0 + github.com/golang-jwt/jwt/v5 v5.3.1 github.com/goproxy/goproxy v0.25.0 github.com/lmittmann/tint v1.1.2 github.com/minio/minio-go/v7 v7.0.97 diff --git a/go.sum b/go.sum index 4fb62b8..2672c3d 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,8 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= diff --git a/internal/config/kong.go b/internal/config/kong.go index e529b39..82cf6ec 100644 --- a/internal/config/kong.go +++ b/internal/config/kong.go @@ -3,6 +3,7 @@ package config import ( "fmt" "io" + "os" "strings" "github.com/alecthomas/hcl/v2" @@ -14,9 +15,24 @@ func KongLoader[GlobalConfig any](r io.Reader) (kong.Resolver, error) { if err != nil { return nil, fmt.Errorf("failed to parse HCL: %w", err) } + + // Expand environment variables in the AST + expandVars(ast, ParseEnvars()) + return &kongResolver{flattenHCL(ast)}, nil } +// ParseEnvars returns a map of all environment variables. +func ParseEnvars() map[string]string { + envars := map[string]string{} + for _, env := range os.Environ() { + if key, value, ok := strings.Cut(env, "="); ok { + envars[key] = value + } + } + return envars +} + type kongResolver struct { values map[string]any } diff --git a/internal/gitclone/command.go b/internal/gitclone/command.go index 31a3a17..5aceda4 100644 --- a/internal/gitclone/command.go +++ b/internal/gitclone/command.go @@ -5,13 +5,23 @@ package gitclone import ( "bufio" "context" + "fmt" "os/exec" "strings" "github.com/alecthomas/errors" ) -func gitCommand(ctx context.Context, url string, args ...string) (*exec.Cmd, error) { +func gitCommand(ctx context.Context, url string, credentialProvider CredentialProvider, args ...string) (*exec.Cmd, error) { + modifiedURL := url + if credentialProvider != nil && strings.Contains(url, "github.com") { + token, err := credentialProvider.GetTokenForURL(ctx, url) + if err == nil && token != "" { + modifiedURL = injectTokenIntoURL(url, token) + } + // If error getting token, fall back to original URL (system credentials) + } + configArgs, err := getInsteadOfDisableArgsForURL(ctx, url) if err != nil { return nil, errors.Wrap(err, "get insteadOf disable args") @@ -23,8 +33,34 @@ func gitCommand(ctx context.Context, url string, args ...string) (*exec.Cmd, err } allArgs = append(allArgs, args...) - cmd := exec.CommandContext(ctx, "git", allArgs...) - return cmd, nil + // Replace URL in args if it was modified for authentication + if modifiedURL != url { + for i, arg := range allArgs { + if arg == url { + allArgs[i] = modifiedURL + } + } + } + + return exec.CommandContext(ctx, "git", allArgs...), nil +} + +// Converts https://github.com/org/repo to https://x-access-token:TOKEN@github.com/org/repo +func injectTokenIntoURL(url, token string) string { + if token == "" { + return url + } + + if strings.HasPrefix(url, "https://github.com/") { + return strings.Replace(url, "https://github.com/", fmt.Sprintf("https://x-access-token:%s@github.com/", token), 1) + } + + // Upgrade http to https when adding token for security + if strings.HasPrefix(url, "http://github.com/") { + return strings.Replace(url, "http://github.com/", fmt.Sprintf("https://x-access-token:%s@github.com/", token), 1) + } + + return url } func getInsteadOfDisableArgsForURL(ctx context.Context, targetURL string) ([]string, error) { diff --git a/internal/gitclone/command_test.go b/internal/gitclone/command_test.go index e700a21..12bc2d3 100644 --- a/internal/gitclone/command_test.go +++ b/internal/gitclone/command_test.go @@ -45,7 +45,7 @@ func TestGetInsteadOfDisableArgsForURL(t *testing.T) { func TestGitCommand(t *testing.T) { ctx := context.Background() - cmd, err := gitCommand(ctx, "https://github.com/user/repo", "version") + cmd, err := gitCommand(ctx, "https://github.com/user/repo", nil, "version") assert.NoError(t, err) assert.NotZero(t, cmd) @@ -59,7 +59,7 @@ func TestGitCommand(t *testing.T) { func TestGitCommandWithEmptyURL(t *testing.T) { ctx := context.Background() - cmd, err := gitCommand(ctx, "", "version") + cmd, err := gitCommand(ctx, "", nil, "version") assert.NoError(t, err) assert.NotZero(t, cmd) diff --git a/internal/gitclone/manager.go b/internal/gitclone/manager.go index f332a2d..ccc5382 100644 --- a/internal/gitclone/manager.go +++ b/internal/gitclone/manager.go @@ -74,24 +74,31 @@ type Config struct { GitConfig GitTuningConfig } +// CredentialProvider provides credentials for git operations. +type CredentialProvider interface { + GetTokenForURL(ctx context.Context, url string) (string, error) +} + type Repository struct { - mu sync.RWMutex - state State - path string - upstreamURL string - lastFetch time.Time - lastRefCheck time.Time - refCheckValid bool - fetchSem chan struct{} + mu sync.RWMutex + state State + path string + upstreamURL string + lastFetch time.Time + lastRefCheck time.Time + refCheckValid bool + fetchSem chan struct{} + credentialProvider CredentialProvider } type Manager struct { - config Config - clones map[string]*Repository - clonesMu sync.RWMutex + config Config + clones map[string]*Repository + clonesMu sync.RWMutex + credentialProvider CredentialProvider } -func NewManager(_ context.Context, config Config) (*Manager, error) { +func NewManager(_ context.Context, config Config, credentialProvider CredentialProvider) (*Manager, error) { if config.RootDir == "" { return nil, errors.New("RootDir is required") } @@ -101,8 +108,9 @@ func NewManager(_ context.Context, config Config) (*Manager, error) { } return &Manager{ - config: config, - clones: make(map[string]*Repository), + config: config, + clones: make(map[string]*Repository), + credentialProvider: credentialProvider, }, nil } @@ -125,10 +133,11 @@ func (m *Manager) GetOrCreate(_ context.Context, upstreamURL string) (*Repositor clonePath := m.clonePathForURL(upstreamURL) repo = &Repository{ - state: StateEmpty, - path: clonePath, - upstreamURL: upstreamURL, - fetchSem: make(chan struct{}, 1), + state: StateEmpty, + path: clonePath, + upstreamURL: upstreamURL, + fetchSem: make(chan struct{}, 1), + credentialProvider: m.credentialProvider, } gitDir := filepath.Join(clonePath, ".git") @@ -152,6 +161,10 @@ func (m *Manager) Config() Config { return m.config } +func (m *Manager) CredentialProvider() CredentialProvider { + return m.credentialProvider +} + func (m *Manager) DiscoverExisting(_ context.Context) ([]*Repository, error) { var discovered []*Repository err := filepath.Walk(m.config.RootDir, func(path string, info os.FileInfo, err error) error { @@ -195,10 +208,11 @@ func (m *Manager) DiscoverExisting(_ context.Context) ([]*Repository, error) { upstreamURL := "https://" + host + "/" + repoPath repo := &Repository{ - state: StateReady, - path: path, - upstreamURL: upstreamURL, - fetchSem: make(chan struct{}, 1), + state: StateReady, + path: path, + upstreamURL: upstreamURL, + fetchSem: make(chan struct{}, 1), + credentialProvider: m.credentialProvider, } repo.fetchSem <- struct{}{} @@ -304,7 +318,7 @@ func (r *Repository) executeClone(ctx context.Context, config Config) error { r.upstreamURL, r.path, } - cmd, err := gitCommand(ctx, r.upstreamURL, args...) + cmd, err := gitCommand(ctx, r.upstreamURL, r.credentialProvider, args...) if err != nil { return errors.Wrap(err, "create git command") } @@ -320,7 +334,7 @@ func (r *Repository) executeClone(ctx context.Context, config Config) error { return errors.Wrapf(err, "configure fetch refspec: %s", string(output)) } - cmd, err = gitCommand(ctx, r.upstreamURL, "-C", r.path, + cmd, err = gitCommand(ctx, r.upstreamURL, r.credentialProvider, "-C", r.path, "-c", "http.postBuffer="+strconv.Itoa(config.GitConfig.PostBuffer), "-c", "http.lowSpeedLimit="+strconv.Itoa(config.GitConfig.LowSpeedLimit), "-c", "http.lowSpeedTime="+strconv.Itoa(int(config.GitConfig.LowSpeedTime.Seconds())), @@ -357,7 +371,7 @@ func (r *Repository) Fetch(ctx context.Context, config Config) error { r.mu.Lock() // #nosec G204 - r.path is controlled by us - cmd, err := gitCommand(ctx, r.upstreamURL, "-C", r.path, + cmd, err := gitCommand(ctx, r.upstreamURL, r.credentialProvider, "-C", r.path, "-c", "http.postBuffer="+strconv.Itoa(config.GitConfig.PostBuffer), "-c", "http.lowSpeedLimit="+strconv.Itoa(config.GitConfig.LowSpeedLimit), "-c", "http.lowSpeedTime="+strconv.Itoa(int(config.GitConfig.LowSpeedTime.Seconds())), @@ -447,7 +461,7 @@ func (r *Repository) GetLocalRefs(ctx context.Context) (map[string]string, error func (r *Repository) GetUpstreamRefs(ctx context.Context) (map[string]string, error) { // #nosec G204 - r.upstreamURL is controlled by us - cmd, err := gitCommand(ctx, r.upstreamURL, "ls-remote", r.upstreamURL) + cmd, err := gitCommand(ctx, r.upstreamURL, r.credentialProvider, "ls-remote", r.upstreamURL) if err != nil { return nil, errors.Wrap(err, "create git command") } diff --git a/internal/gitclone/manager_test.go b/internal/gitclone/manager_test.go index f538ae9..b115daa 100644 --- a/internal/gitclone/manager_test.go +++ b/internal/gitclone/manager_test.go @@ -21,7 +21,7 @@ func TestNewManager(t *testing.T) { GitConfig: DefaultGitTuningConfig(), } - manager, err := NewManager(context.Background(), config) + manager, err := NewManager(context.Background(), config, nil) assert.NoError(t, err) assert.NotZero(t, manager) assert.Equal(t, tmpDir, manager.config.RootDir) @@ -33,7 +33,7 @@ func TestNewManager_RequiresRootDir(t *testing.T) { RefCheckInterval: 10 * time.Second, } - _, err := NewManager(context.Background(), config) + _, err := NewManager(context.Background(), config, nil) assert.Error(t, err) assert.Contains(t, err.Error(), "RootDir is required") } @@ -47,7 +47,7 @@ func TestManager_GetOrCreate(t *testing.T) { GitConfig: DefaultGitTuningConfig(), } - manager, err := NewManager(context.Background(), config) + manager, err := NewManager(context.Background(), config, nil) assert.NoError(t, err) upstreamURL := "https://github.com/user/repo" @@ -73,7 +73,7 @@ func TestManager_GetOrCreate_ExistingClone(t *testing.T) { GitConfig: DefaultGitTuningConfig(), } - manager, err := NewManager(context.Background(), config) + manager, err := NewManager(context.Background(), config, nil) assert.NoError(t, err) repoPath := filepath.Join(tmpDir, "github.com", "user", "repo") @@ -98,7 +98,7 @@ func TestManager_Get(t *testing.T) { GitConfig: DefaultGitTuningConfig(), } - manager, err := NewManager(context.Background(), config) + manager, err := NewManager(context.Background(), config, nil) assert.NoError(t, err) upstreamURL := "https://github.com/user/repo" @@ -123,7 +123,7 @@ func TestManager_DiscoverExisting(t *testing.T) { GitConfig: DefaultGitTuningConfig(), } - manager, err := NewManager(context.Background(), config) + manager, err := NewManager(context.Background(), config, nil) assert.NoError(t, err) repos := []string{ diff --git a/internal/githubapp/config.go b/internal/githubapp/config.go new file mode 100644 index 0000000..b9927af --- /dev/null +++ b/internal/githubapp/config.go @@ -0,0 +1,70 @@ +// Package githubapp provides GitHub App authentication and token management. +package githubapp + +import ( + "encoding/json" + "log/slog" + "time" + + "github.com/alecthomas/errors" +) + +type Config struct { + AppID string `hcl:"app-id" help:"GitHub App ID"` + PrivateKeyPath string `hcl:"private-key-path" help:"Path to GitHub App private key (PEM format)"` + InstallationsJSON string `hcl:"installations-json" help:"JSON string mapping org names to installation IDs"` + + // Populated from InstallationsJSON during Initialize(). Not exposed in HCL. + Installations map[string]string +} + +type TokenCacheConfig struct { + RefreshBuffer time.Duration // How early to refresh before expiration + JWTExpiration time.Duration // GitHub allows max 10 minutes +} + +func DefaultTokenCacheConfig() TokenCacheConfig { + return TokenCacheConfig{ + RefreshBuffer: 5 * time.Minute, + JWTExpiration: 10 * time.Minute, + } +} + +// Initialize must be called after loading config to parse InstallationsJSON into Installations map. +func (c *Config) Initialize(logger *slog.Logger) error { + if c.InstallationsJSON == "" { + return errors.New("installations-json is required") + } + + var installations map[string]string + if err := json.Unmarshal([]byte(c.InstallationsJSON), &installations); err != nil { + logger.Error("Failed to parse installations-json", + "error", err, + "installations_json", c.InstallationsJSON) + return errors.Wrap(err, "parse installations-json") + } + + if len(installations) == 0 { + return errors.New("installations-json must contain at least one organization") + } + + c.Installations = installations + + logger.Info("GitHub App config initialized", + "app_id", c.AppID, + "private_key_path", c.PrivateKeyPath, + "installations_json", c.InstallationsJSON) + + return nil +} + +func (c *Config) IsConfigured() bool { + return c.AppID != "" && c.PrivateKeyPath != "" && len(c.Installations) > 0 +} + +func (c *Config) GetInstallationID(org string) string { + if c.Installations == nil { + return "" + } + return c.Installations[org] +} diff --git a/internal/githubapp/jwt.go b/internal/githubapp/jwt.go new file mode 100644 index 0000000..a2b5edb --- /dev/null +++ b/internal/githubapp/jwt.go @@ -0,0 +1,78 @@ +package githubapp + +import ( + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "os" + "time" + + "github.com/alecthomas/errors" + "github.com/golang-jwt/jwt/v5" +) + +type JWTGenerator struct { + appID string + privateKey *rsa.PrivateKey + expiration time.Duration +} + +func NewJWTGenerator(appID, privateKeyPath string, expiration time.Duration) (*JWTGenerator, error) { + privateKey, err := loadPrivateKey(privateKeyPath) + if err != nil { + return nil, errors.Wrap(err, "load private key") + } + + return &JWTGenerator{ + appID: appID, + privateKey: privateKey, + expiration: expiration, + }, nil +} + +func (g *JWTGenerator) GenerateJWT() (string, error) { + now := time.Now() + + claims := jwt.RegisteredClaims{ + IssuedAt: jwt.NewNumericDate(now), + ExpiresAt: jwt.NewNumericDate(now.Add(g.expiration)), + Issuer: g.appID, + } + + token := jwt.NewWithClaims(jwt.SigningMethodRS256, claims) + signedToken, err := token.SignedString(g.privateKey) + if err != nil { + return "", errors.Wrap(err, "sign JWT") + } + + return signedToken, nil +} + +func loadPrivateKey(path string) (*rsa.PrivateKey, error) { + keyData, err := os.ReadFile(path) + if err != nil { + return nil, errors.Wrapf(err, "read private key file: %s", path) + } + + block, _ := pem.Decode(keyData) + if block == nil { + return nil, errors.Errorf("failed to decode PEM block from private key file: %s", path) + } + + privateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes) + if err == nil { + return privateKey, nil + } + + key, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return nil, errors.Wrap(err, "parse private key (tried both PKCS1 and PKCS8)") + } + + rsaKey, ok := key.(*rsa.PrivateKey) + if !ok { + return nil, errors.Errorf("private key is not RSA (type: %T)", key) + } + + return rsaKey, nil +} diff --git a/internal/githubapp/tokens.go b/internal/githubapp/tokens.go new file mode 100644 index 0000000..115ea38 --- /dev/null +++ b/internal/githubapp/tokens.go @@ -0,0 +1,176 @@ +package githubapp + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "strings" + "sync" + "time" + + "github.com/alecthomas/errors" + + "github.com/block/cachew/internal/logging" +) + +var ( + sharedTokenManager *TokenManager + sharedTokenManagerMu sync.RWMutex +) + +func SetShared(tm *TokenManager) { + sharedTokenManagerMu.Lock() + defer sharedTokenManagerMu.Unlock() + sharedTokenManager = tm +} + +func GetShared() *TokenManager { + sharedTokenManagerMu.RLock() + defer sharedTokenManagerMu.RUnlock() + return sharedTokenManager +} + +type TokenManager struct { + config Config + cacheConfig TokenCacheConfig + jwtGenerator *JWTGenerator + httpClient *http.Client + + mu sync.RWMutex + tokens map[string]*cachedToken +} + +type cachedToken struct { + token string + expiresAt time.Time +} + +func NewTokenManager(config Config, cacheConfig TokenCacheConfig) (*TokenManager, error) { + if !config.IsConfigured() { + return nil, errors.New("GitHub App not configured") + } + + jwtGenerator, err := NewJWTGenerator(config.AppID, config.PrivateKeyPath, cacheConfig.JWTExpiration) + if err != nil { + return nil, errors.Wrap(err, "create JWT generator") + } + + return &TokenManager{ + config: config, + cacheConfig: cacheConfig, + jwtGenerator: jwtGenerator, + httpClient: http.DefaultClient, + tokens: make(map[string]*cachedToken), + }, nil +} + +func (tm *TokenManager) GetTokenForOrg(ctx context.Context, org string) (string, error) { + if tm == nil { + return "", errors.New("token manager not initialized") + } + logger := logging.FromContext(ctx).With(slog.String("org", org)) + + installationID := tm.config.GetInstallationID(org) + if installationID == "" { + return "", errors.Errorf("no GitHub App installation configured for org: %s", org) + } + + tm.mu.RLock() + cached, exists := tm.tokens[org] + tm.mu.RUnlock() + + if exists && time.Now().Add(tm.cacheConfig.RefreshBuffer).Before(cached.expiresAt) { + logger.DebugContext(ctx, "Using cached GitHub App token") + return cached.token, nil + } + + logger.DebugContext(ctx, "Fetching new GitHub App installation token", + slog.String("installation_id", installationID)) + + token, expiresAt, err := tm.fetchInstallationToken(ctx, installationID) + if err != nil { + return "", errors.Wrap(err, "fetch installation token") + } + + tm.mu.Lock() + tm.tokens[org] = &cachedToken{ + token: token, + expiresAt: expiresAt, + } + tm.mu.Unlock() + + logger.InfoContext(ctx, "GitHub App token refreshed", + slog.Time("expires_at", expiresAt)) + + return token, nil +} + +func (tm *TokenManager) GetTokenForURL(ctx context.Context, url string) (string, error) { + if tm == nil { + return "", errors.New("token manager not initialized") + } + org, err := extractOrgFromURL(url) + if err != nil { + return "", err + } + + return tm.GetTokenForOrg(ctx, org) +} + +func (tm *TokenManager) fetchInstallationToken(ctx context.Context, installationID string) (string, time.Time, error) { + jwt, err := tm.jwtGenerator.GenerateJWT() + if err != nil { + return "", time.Time{}, errors.Wrap(err, "generate JWT") + } + + url := fmt.Sprintf("https://api.github.com/app/installations/%s/access_tokens", installationID) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, nil) + if err != nil { + return "", time.Time{}, errors.Wrap(err, "create request") + } + + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("Authorization", "Bearer "+jwt) + req.Header.Set("X-Github-Api-Version", "2022-11-28") + + resp, err := tm.httpClient.Do(req) + if err != nil { + return "", time.Time{}, errors.Wrap(err, "execute request") + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + return "", time.Time{}, errors.Errorf("GitHub API returned status %d", resp.StatusCode) + } + + var result struct { + Token string `json:"token"` + ExpiresAt time.Time `json:"expires_at"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return "", time.Time{}, errors.Wrap(err, "decode response") + } + + return result.Token, result.ExpiresAt, nil +} + +func extractOrgFromURL(url string) (string, error) { + url = strings.TrimPrefix(url, "https://") + url = strings.TrimPrefix(url, "http://") + url = strings.TrimPrefix(url, "git@") + + if !strings.HasPrefix(url, "github.com/") && !strings.HasPrefix(url, "github.com:") { + return "", errors.Errorf("not a GitHub URL: %s", url) + } + url = strings.TrimPrefix(url, "github.com/") + url = strings.TrimPrefix(url, "github.com:") + + parts := strings.Split(url, "/") + if len(parts) < 1 || parts[0] == "" { + return "", errors.Errorf("cannot extract org from URL: %s", url) + } + + return parts[0], nil +} diff --git a/internal/strategy/git/git.go b/internal/strategy/git/git.go index 8c2fb6e..56e9474 100644 --- a/internal/strategy/git/git.go +++ b/internal/strategy/git/git.go @@ -21,6 +21,7 @@ import ( "github.com/block/cachew/internal/cache" "github.com/block/cachew/internal/gitclone" + "github.com/block/cachew/internal/githubapp" "github.com/block/cachew/internal/jobscheduler" "github.com/block/cachew/internal/logging" "github.com/block/cachew/internal/strategy" @@ -38,6 +39,7 @@ type Config struct { RefCheckInterval time.Duration `hcl:"ref-check-interval,optional" help:"How long to cache ref checks." default:"10s"` BundleInterval time.Duration `hcl:"bundle-interval,optional" help:"How often to generate bundles. 0 disables bundling." default:"0"` SnapshotInterval time.Duration `hcl:"snapshot-interval,optional" help:"How often to generate tar.zstd snapshots. 0 disables snapshots." default:"0"` + CloneDepth int `hcl:"clone-depth,optional" help:"Depth for shallow clones. 0 means full clone." default:"0"` } type Strategy struct { @@ -50,6 +52,7 @@ type Strategy struct { scheduler jobscheduler.Scheduler spoolsMu sync.Mutex spools map[string]*RepoSpools + tokenManager *githubapp.TokenManager } func New(ctx context.Context, config Config, scheduler jobscheduler.Scheduler, cache cache.Cache, mux strategy.Mux) (*Strategy, error) { @@ -67,12 +70,20 @@ func New(ctx context.Context, config Config, scheduler jobscheduler.Scheduler, c config.RefCheckInterval = 10 * time.Second } + // Use shared GitHub App token manager if configured + tokenManager := githubapp.GetShared() + if tokenManager != nil { + logger.InfoContext(ctx, "Using global GitHub App authentication for git strategy") + } else { + logger.WarnContext(ctx, "GitHub App not configured, using system git credentials") + } + cloneManager, err := gitclone.NewManager(ctx, gitclone.Config{ RootDir: config.MirrorRoot, FetchInterval: config.FetchInterval, RefCheckInterval: config.RefCheckInterval, GitConfig: gitclone.DefaultGitTuningConfig(), - }) + }, tokenManager) if err != nil { return nil, errors.Wrap(err, "create clone manager") } @@ -91,6 +102,7 @@ func New(ctx context.Context, config Config, scheduler jobscheduler.Scheduler, c ctx: ctx, scheduler: scheduler.WithQueuePrefix("git"), spools: make(map[string]*RepoSpools), + tokenManager: tokenManager, } existing, err := s.cloneManager.DiscoverExisting(ctx) @@ -113,6 +125,22 @@ func New(ctx context.Context, config Config, scheduler jobscheduler.Scheduler, c req.URL.Host = req.PathValue("host") req.URL.Path = "/" + req.PathValue("path") req.Host = req.URL.Host + + // Inject GitHub App authentication for github.com requests + if s.tokenManager != nil && req.URL.Host == "github.com" { + // Extract org from path (e.g., /squareup/blox.git/...) + parts := strings.Split(strings.TrimPrefix(req.URL.Path, "/"), "/") + if len(parts) >= 1 && parts[0] != "" { + org := parts[0] + token, err := s.tokenManager.GetTokenForOrg(req.Context(), org) + if err == nil && token != "" { + // Inject token as Basic auth with "x-access-token" username + req.SetBasicAuth("x-access-token", token) + logger.DebugContext(req.Context(), "Injecting GitHub App auth into upstream request", + slog.String("org", org)) + } + } + } }, Transport: s.httpClient.Transport, ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) { diff --git a/internal/strategy/github_releases.go b/internal/strategy/github_releases.go index fa2da49..47f1419 100644 --- a/internal/strategy/github_releases.go +++ b/internal/strategy/github_releases.go @@ -11,6 +11,7 @@ import ( "github.com/alecthomas/errors" "github.com/block/cachew/internal/cache" + "github.com/block/cachew/internal/githubapp" "github.com/block/cachew/internal/httputil" "github.com/block/cachew/internal/logging" "github.com/block/cachew/internal/strategy/handler" @@ -21,27 +22,35 @@ func RegisterGitHubReleases(r *Registry) { } type GitHubReleasesConfig struct { - Token string `hcl:"token" help:"GitHub token for authentication."` + Token string `hcl:"token,optional" help:"GitHub token for authentication."` PrivateOrgs []string `hcl:"private-orgs" help:"List of private GitHub organisations."` } // The GitHubReleases strategy fetches private (and public) release binaries from GitHub. type GitHubReleases struct { - config GitHubReleasesConfig - cache cache.Cache - client *http.Client + config GitHubReleasesConfig + cache cache.Cache + client *http.Client + tokenManager *githubapp.TokenManager } // NewGitHubReleases creates a [Strategy] that fetches private (and public) release binaries from GitHub. func NewGitHubReleases(ctx context.Context, config GitHubReleasesConfig, cache cache.Cache, mux Mux) (*GitHubReleases, error) { - s := &GitHubReleases{ - config: config, - cache: cache, - client: http.DefaultClient, - } logger := logging.FromContext(ctx) - if config.Token == "" { - logger.WarnContext(ctx, "No token configured for github-releases strategy") + + // Use shared GitHub App token manager if configured + tokenManager := githubapp.GetShared() + if tokenManager != nil { + logger.InfoContext(ctx, "Using global GitHub App authentication for private github-releases strategy") + } else if config.Token == "" { + logger.WarnContext(ctx, "No authentication configured for github-releases strategy") + } + + s := &GitHubReleases{ + config: config, + cache: cache, + client: http.DefaultClient, + tokenManager: tokenManager, } // eg. https://github.com/alecthomas/chroma/releases/download/v2.21.1/chroma-2.21.1-darwin-amd64.tar.gz h := handler.New(s.client, cache). @@ -68,16 +77,31 @@ var _ Strategy = (*GitHubReleases)(nil) func (g *GitHubReleases) String() string { return "github-releases" } // newGitHubRequest creates a new HTTP request with GitHub API headers and authentication. -func (g *GitHubReleases) newGitHubRequest(ctx context.Context, url, accept string) (*http.Request, error) { +func (g *GitHubReleases) newGitHubRequest(ctx context.Context, url, accept, org string) (*http.Request, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return nil, errors.Wrap(err, "create request") } req.Header.Set("Accept", accept) req.Header.Set("X-Github-Api-Version", "2022-11-28") - if g.config.Token != "" { + + // Try GitHub App authentication first, fall back to static token + if g.tokenManager != nil && org != "" { + token, err := g.tokenManager.GetTokenForOrg(ctx, org) + if err != nil { + logging.FromContext(ctx).WarnContext(ctx, "Failed to get GitHub App token, falling back to static token", + slog.String("org", org), + slog.String("error", err.Error())) + } else if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + } + + // Fall back to static token if GitHub App not used or failed + if req.Header.Get("Authorization") == "" && g.config.Token != "" { req.Header.Set("Authorization", "Bearer "+g.config.Token) } + return req, nil } @@ -107,7 +131,7 @@ func (g *GitHubReleases) downloadRelease(ctx context.Context, org, repo, release // Use GitHub API to get release info and find the asset logger.DebugContext(ctx, "Using GitHub API for private release") apiURL := fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/tags/%s", org, repo, release) - req, err := g.newGitHubRequest(ctx, apiURL, "application/vnd.github+json") + req, err := g.newGitHubRequest(ctx, apiURL, "application/vnd.github+json", org) if err != nil { return nil, httputil.Errorf(http.StatusInternalServerError, "create API request") } @@ -148,7 +172,7 @@ func (g *GitHubReleases) downloadRelease(ctx context.Context, org, repo, release logger.DebugContext(ctx, "Found asset in release", slog.String("asset_url", assetURL)) // Create request for the asset download - req, err = g.newGitHubRequest(ctx, assetURL, "application/octet-stream") + req, err = g.newGitHubRequest(ctx, assetURL, "application/octet-stream", org) if err != nil { return nil, httputil.Errorf(http.StatusInternalServerError, "create asset request failed: %w", err) }