Author: Honza Pokorny <me@honza.ca>
Implement git cloning
include/repo-index.html | 5 pkg/go-git-http/.gitignore | 23 ++ pkg/go-git-http/.travis.yml | 5 pkg/go-git-http/README.md | 4 pkg/go-git-http/auth/auth.go | 106 ++++++++++ pkg/go-git-http/auth/auth_test.go | 15 + pkg/go-git-http/auth/basicauth.go | 47 ++++ pkg/go-git-http/auth/basicauth_test.go | 28 ++ pkg/go-git-http/errors.go | 14 + pkg/go-git-http/events.go | 81 +++++++ pkg/go-git-http/git_reader.go | 49 ++++ pkg/go-git-http/githttp.go | 293 ++++++++++++++++++++++++++++ pkg/go-git-http/pktparser.go | 143 +++++++++++++ pkg/go-git-http/pktparser_test.go | 44 ++++ pkg/go-git-http/routing.go | 120 +++++++++++ pkg/go-git-http/rpc_reader.go | 120 +++++++++++ pkg/go-git-http/rpc_reader_test.go | 193 ++++++++++++++++++ pkg/go-git-http/testdata/upload-pack.0 | 34 +++ pkg/go-git-http/testdata/upload-pack.1 | 3 pkg/go-git-http/utils.go | 93 ++++++++ pkg/go-git-http/version.go | 3 pkg/smithy/smithy.go | 14 +
diff --git a/include/repo-index.html b/include/repo-index.html index 21c07e9c9b9bbf64d74394a053a6f059a59f63ee..905954c96682e48938798320add5bb0240b7beb2 100644 --- a/include/repo-index.html +++ b/include/repo-index.html @@ -26,6 +26,11 @@ <div class="row"> <div class="col-xl-6 col-lg-6 col-md-12 col-sm-12"> {{ .Readme }} + + <hr> + <pre> +$ git clone https://{{ .Host }}/git/{{ $repo }} + </pre> </div> </div> diff --git a/pkg/go-git-http/.gitignore b/pkg/go-git-http/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..836562412fe8a44fa99a515eeff68d2bc1a86daa --- /dev/null +++ b/pkg/go-git-http/.gitignore @@ -0,0 +1,23 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test diff --git a/pkg/go-git-http/.travis.yml b/pkg/go-git-http/.travis.yml new file mode 100644 index 0000000000000000000000000000000000000000..f4f458a416d448aada59a12c7f41a65055f6be51 --- /dev/null +++ b/pkg/go-git-http/.travis.yml @@ -0,0 +1,5 @@ +language: go + +go: + - 1.5 + - tip diff --git a/pkg/go-git-http/README.md b/pkg/go-git-http/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d17010c92c8e9bb732329263d5ae912f2147a9e6 --- /dev/null +++ b/pkg/go-git-http/README.md @@ -0,0 +1,4 @@ +Source: https://github.com/AaronO/go-git-http +Original code released under Apache License 2.0 + +Changes: add "git/" prefix to all routes diff --git a/pkg/go-git-http/auth/auth.go b/pkg/go-git-http/auth/auth.go new file mode 100644 index 0000000000000000000000000000000000000000..6df0aef36888385556d5cf900bbba61e6aedcaba --- /dev/null +++ b/pkg/go-git-http/auth/auth.go @@ -0,0 +1,106 @@ +package auth + +import ( + "net/http" + "regexp" + "strings" +) + +type AuthInfo struct { + // Usernane or email + Username string + // Plaintext password or token + Password string + + // repo component of URL + // Usually: "username/repo_name" + // But could also be: "some_repo.git" + Repo string + + // Are we pushing or fetching ? + Push bool + Fetch bool +} + +var ( + repoNameRegex = regexp.MustCompile("^/?(.*?)/(HEAD|git-upload-pack|git-receive-pack|info/refs|objects/.*)$") +) + +func Authenticator(authf func(AuthInfo) (bool, error)) func(http.Handler) http.Handler { + return func(handler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + auth, err := parseAuthHeader(req.Header.Get("Authorization")) + if err != nil { + w.Header().Set("WWW-Authenticate", `Basic realm="git server"`) + http.Error(w, err.Error(), 401) + return + } + + // Build up info from request headers and URL + info := AuthInfo{ + Username: auth.Name, + Password: auth.Pass, + Repo: repoName(req.URL.Path), + Push: isPush(req), + Fetch: isFetch(req), + } + + // Call authentication function + authenticated, err := authf(info) + if err != nil { + code := 500 + msg := err.Error() + if se, ok := err.(StatusError); ok { + code = se.StatusCode() + } + http.Error(w, msg, code) + return + } + + // Deny access to repo + if !authenticated { + http.Error(w, "Forbidden", 403) + return + } + + // Access granted + handler.ServeHTTP(w, req) + }) + } +} + +func isFetch(req *http.Request) bool { + return isService("upload-pack", req) +} + +func isPush(req *http.Request) bool { + return isService("receive-pack", req) +} + +func isService(service string, req *http.Request) bool { + return getServiceType(req) == service || strings.HasSuffix(req.URL.Path, service) +} + +func repoName(urlPath string) string { + matches := repoNameRegex.FindStringSubmatch(urlPath) + if matches == nil { + return "" + } + return matches[1] +} + +func getServiceType(r *http.Request) string { + service_type := r.FormValue("service") + + if s := strings.HasPrefix(service_type, "git-"); !s { + return "" + } + + return strings.Replace(service_type, "git-", "", 1) +} + +// StatusCode is an interface allowing authenticators +// to pass down error's with an http error code +type StatusError interface { + StatusCode() int +} diff --git a/pkg/go-git-http/auth/auth_test.go b/pkg/go-git-http/auth/auth_test.go new file mode 100644 index 0000000000000000000000000000000000000000..f3b3510d0b824b466d986a86492d07e30405e581 --- /dev/null +++ b/pkg/go-git-http/auth/auth_test.go @@ -0,0 +1,15 @@ +package auth + +import ( + "testing" +) + +func TestRepoName(t *testing.T) { + if x := repoName("/yapp.ss.git/HEAD"); x != "yapp.ss.git" { + t.Errorf("Should have been 'yapp.js.git' is '%s'", x) + } + + if x := repoName("aarono/gogo-proxy/HEAD"); x != "aarono/gogo-proxy" { + t.Errorf("Should have been 'aarono/gogo-proxy' is '%s'", x) + } +} diff --git a/pkg/go-git-http/auth/basicauth.go b/pkg/go-git-http/auth/basicauth.go new file mode 100644 index 0000000000000000000000000000000000000000..44593a92586ebe36a34cdaa75fa5ec80469abd39 --- /dev/null +++ b/pkg/go-git-http/auth/basicauth.go @@ -0,0 +1,47 @@ +package auth + +import ( + "encoding/base64" + "fmt" + "regexp" + "strings" +) + +// Parse http basic header +type BasicAuth struct { + Name string + Pass string +} + +var ( + basicAuthRegex = regexp.MustCompile("^([^:]*):(.*)$") +) + +func parseAuthHeader(header string) (*BasicAuth, error) { + parts := strings.SplitN(header, " ", 2) + if len(parts) < 2 { + return nil, fmt.Errorf("Invalid authorization header, not enought parts") + } + + authType := parts[0] + authData := parts[1] + + if strings.ToLower(authType) != "basic" { + return nil, fmt.Errorf("Authentication '%s' was not of 'Basic' type", authType) + } + + data, err := base64.StdEncoding.DecodeString(authData) + if err != nil { + return nil, err + } + + matches := basicAuthRegex.FindStringSubmatch(string(data)) + if matches == nil { + return nil, fmt.Errorf("Authorization data '%s' did not match auth regexp", data) + } + + return &BasicAuth{ + Name: matches[1], + Pass: matches[2], + }, nil +} diff --git a/pkg/go-git-http/auth/basicauth_test.go b/pkg/go-git-http/auth/basicauth_test.go new file mode 100644 index 0000000000000000000000000000000000000000..256d6d33098734a724053b7dd869e2e4ea54e6c4 --- /dev/null +++ b/pkg/go-git-http/auth/basicauth_test.go @@ -0,0 +1,28 @@ +package auth + +import ( + "testing" +) + +func TestHeaderParsing(t *testing.T) { + // Basic admin:password + authorization := "Basic YWRtaW46cGFzc3dvcmQ=" + + auth, err := parseAuthHeader(authorization) + if err != nil { + t.Error(err) + } + + if auth.Name != "admin" { + t.Errorf("Detected name does not match: '%s'", auth.Name) + } + if auth.Pass != "password" { + t.Errorf("Detected password does not match: '%s'", auth.Pass) + } +} + +func TestEmptyHeader(t *testing.T) { + if _, err := parseAuthHeader(""); err == nil { + t.Errorf("Empty headers should generate errors") + } +} diff --git a/pkg/go-git-http/errors.go b/pkg/go-git-http/errors.go new file mode 100644 index 0000000000000000000000000000000000000000..4a56db26dd96855ced1cf9f2ce6f551c9630d3f7 --- /dev/null +++ b/pkg/go-git-http/errors.go @@ -0,0 +1,14 @@ +package githttp + +import ( + "fmt" +) + +type ErrorNoAccess struct { + // Path to directory of repo accessed + Dir string +} + +func (e *ErrorNoAccess) Error() string { + return fmt.Sprintf("Could not access repo at '%s'", e.Dir) +} diff --git a/pkg/go-git-http/events.go b/pkg/go-git-http/events.go new file mode 100644 index 0000000000000000000000000000000000000000..c8b577b1a6cd8225f088c487c305d415a29146c8 --- /dev/null +++ b/pkg/go-git-http/events.go @@ -0,0 +1,81 @@ +package githttp + +import ( + "fmt" + "net/http" +) + +// An event (triggered on push/pull) +type Event struct { + // One of tag/push/fetch + Type EventType `json:"type"` + + //// + // Set for pushes and pulls + //// + + // SHA of commit + Commit string `json:"commit"` + + // Path to bare repo + Dir string + + //// + // Set for pushes or tagging + //// + Tag string `json:"tag,omitempty"` + Last string `json:"last,omitempty"` + Branch string `json:"branch,omitempty"` + + // Error contains the error that happened (if any) + // during this action/event + Error error + + // Http stuff + Request *http.Request +} + +type EventType int + +// Possible event types +const ( + TAG = iota + 1 + PUSH + FETCH + PUSH_FORCE +) + +func (e EventType) String() string { + switch e { + case TAG: + return "tag" + case PUSH: + return "push" + case PUSH_FORCE: + return "push-force" + case FETCH: + return "fetch" + } + return "unknown" +} + +func (e EventType) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf(`"%s"`, e)), nil +} + +func (e EventType) UnmarshalJSON(data []byte) error { + str := string(data[:]) + switch str { + case "tag": + e = TAG + case "push": + e = PUSH + case "push-force": + e = PUSH_FORCE + case "fetch": + e = FETCH + default: + return fmt.Errorf("'%s' is not a known git event type") + } + return nil +} diff --git a/pkg/go-git-http/git_reader.go b/pkg/go-git-http/git_reader.go new file mode 100644 index 0000000000000000000000000000000000000000..354568d2c4a73002a22072e8c810abcde41c1fa8 --- /dev/null +++ b/pkg/go-git-http/git_reader.go @@ -0,0 +1,49 @@ +package githttp + +import ( + "errors" + "io" + "regexp" +) + +// GitReader scans for errors in the output of a git command +type GitReader struct { + // Underlying reader (to relay calls to) + io.Reader + + // Error + GitError error +} + +// Regex to detect errors +var ( + gitErrorRegex = regexp.MustCompile("error: (.*)") +) + +// Implement the io.Reader interface +func (g *GitReader) Read(p []byte) (n int, err error) { + // Relay call + n, err = g.Reader.Read(p) + + // Scan for errors + g.scan(p) + + return n, err +} + +func (g *GitReader) scan(data []byte) { + // Already got an error + // the main error will be the first error line + if g.GitError != nil { + return + } + + matches := gitErrorRegex.FindSubmatch(data) + + // Skip, no matches found + if matches == nil { + return + } + + g.GitError = errors.New(string(matches[1][:])) +} diff --git a/pkg/go-git-http/githttp.go b/pkg/go-git-http/githttp.go new file mode 100644 index 0000000000000000000000000000000000000000..e87ceafb86ed90f21a84c0f60addb6cf0085b519 --- /dev/null +++ b/pkg/go-git-http/githttp.go @@ -0,0 +1,293 @@ +package githttp + +import ( + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path" + "strings" +) + +type GitHttp struct { + // Root directory to serve repos from + ProjectRoot string + + // Path to git binary + GitBinPath string + + // Access rules + UploadPack bool + ReceivePack bool + + // Event handling functions + EventHandler func(ev Event) +} + +// Implement the http.Handler interface +func (g *GitHttp) ServeHTTP(w http.ResponseWriter, r *http.Request) { + g.requestHandler(w, r) + return +} + +// Shorthand constructor for most common scenario +func New(root string) *GitHttp { + return &GitHttp{ + ProjectRoot: root, + GitBinPath: "/usr/bin/git", + UploadPack: true, + ReceivePack: true, + } +} + +// Build root directory if doesn't exist +func (g *GitHttp) Init() (*GitHttp, error) { + if err := os.MkdirAll(g.ProjectRoot, os.ModePerm); err != nil { + return nil, err + } + return g, nil +} + +// Publish event if EventHandler is set +func (g *GitHttp) event(e Event) { + if g.EventHandler != nil { + g.EventHandler(e) + } else { + fmt.Printf("EVENT: %q\n", e) + } +} + +// Actual command handling functions + +func (g *GitHttp) serviceRpc(hr HandlerReq) error { + w, r, rpc, dir := hr.w, hr.r, hr.Rpc, hr.Dir + + access, err := g.hasAccess(r, dir, rpc, true) + if err != nil { + return err + } + + if access == false { + return &ErrorNoAccess{hr.Dir} + } + + // Reader that decompresses if necessary + reader, err := requestReader(r) + if err != nil { + return err + } + defer reader.Close() + + // Reader that scans for events + rpcReader := &RpcReader{ + Reader: reader, + Rpc: rpc, + } + + // Set content type + w.Header().Set("Content-Type", fmt.Sprintf("application/x-git-%s-result", rpc)) + + args := []string{rpc, "--stateless-rpc", "."} + cmd := exec.Command(g.GitBinPath, args...) + cmd.Dir = dir + stdin, err := cmd.StdinPipe() + if err != nil { + return err + } + + stdout, err := cmd.StdoutPipe() + if err != nil { + return err + } + defer stdout.Close() + + err = cmd.Start() + if err != nil { + return err + } + + // Scan's git command's output for errors + gitReader := &GitReader{ + Reader: stdout, + } + + // Copy input to git binary + io.Copy(stdin, rpcReader) + stdin.Close() + + // Write git binary's output to http response + io.Copy(w, gitReader) + + // Wait till command has completed + mainError := cmd.Wait() + + if mainError == nil { + mainError = gitReader.GitError + } + + // Fire events + for _, e := range rpcReader.Events { + // Set directory to current repo + e.Dir = dir + e.Request = hr.r + e.Error = mainError + + // Fire event + g.event(e) + } + + // Because a response was already written, + // the header cannot be changed + return nil +} + +func (g *GitHttp) getInfoRefs(hr HandlerReq) error { + w, r, dir := hr.w, hr.r, hr.Dir + service_name := getServiceType(r) + access, err := g.hasAccess(r, dir, service_name, false) + if err != nil { + return err + } + + if !access { + g.updateServerInfo(dir) + hdrNocache(w) + return sendFile("text/plain; charset=utf-8", hr) + } + + args := []string{service_name, "--stateless-rpc", "--advertise-refs", "."} + refs, err := g.gitCommand(dir, args...) + if err != nil { + return err + } + + hdrNocache(w) + w.Header().Set("Content-Type", fmt.Sprintf("application/x-git-%s-advertisement", service_name)) + w.WriteHeader(http.StatusOK) + w.Write(packetWrite("# service=git-" + service_name + "\n")) + w.Write(packetFlush()) + w.Write(refs) + + return nil +} + +func (g *GitHttp) getInfoPacks(hr HandlerReq) error { + hdrCacheForever(hr.w) + return sendFile("text/plain; charset=utf-8", hr) +} + +func (g *GitHttp) getLooseObject(hr HandlerReq) error { + hdrCacheForever(hr.w) + return sendFile("application/x-git-loose-object", hr) +} + +func (g *GitHttp) getPackFile(hr HandlerReq) error { + hdrCacheForever(hr.w) + return sendFile("application/x-git-packed-objects", hr) +} + +func (g *GitHttp) getIdxFile(hr HandlerReq) error { + hdrCacheForever(hr.w) + return sendFile("application/x-git-packed-objects-toc", hr) +} + +func (g *GitHttp) getTextFile(hr HandlerReq) error { + hdrNocache(hr.w) + return sendFile("text/plain", hr) +} + +// Logic helping functions + +func sendFile(content_type string, hr HandlerReq) error { + w, r := hr.w, hr.r + req_file := path.Join(hr.Dir, hr.File) + + f, err := os.Stat(req_file) + if err != nil { + return err + } + + w.Header().Set("Content-Type", content_type) + w.Header().Set("Content-Length", fmt.Sprintf("%d", f.Size())) + w.Header().Set("Last-Modified", f.ModTime().Format(http.TimeFormat)) + http.ServeFile(w, r, req_file) + + return nil +} + +func (g *GitHttp) getGitDir(file_path string) (string, error) { + root := g.ProjectRoot + + if root == "" { + cwd, err := os.Getwd() + + if err != nil { + return "", err + } + + root = cwd + } + + f := path.Join(root, file_path) + if _, err := os.Stat(f); os.IsNotExist(err) { + return "", err + } + + return f, nil +} + +func (g *GitHttp) hasAccess(r *http.Request, dir string, rpc string, check_content_type bool) (bool, error) { + if check_content_type { + if r.Header.Get("Content-Type") != fmt.Sprintf("application/x-git-%s-request", rpc) { + return false, nil + } + } + + if !(rpc == "upload-pack" || rpc == "receive-pack") { + return false, nil + } + if rpc == "receive-pack" { + return g.ReceivePack, nil + } + if rpc == "upload-pack" { + return g.UploadPack, nil + } + + return g.getConfigSetting(rpc, dir) +} + +func (g *GitHttp) getConfigSetting(service_name string, dir string) (bool, error) { + service_name = strings.Replace(service_name, "-", "", -1) + setting, err := g.getGitConfig("http."+service_name, dir) + if err != nil { + return false, nil + } + + if service_name == "uploadpack" { + return setting != "false", nil + } + + return setting == "true", nil +} + +func (g *GitHttp) getGitConfig(config_name string, dir string) (string, error) { + args := []string{"config", config_name} + out, err := g.gitCommand(dir, args...) + if err != nil { + return "", err + } + return string(out)[0 : len(out)-1], nil +} + +func (g *GitHttp) updateServerInfo(dir string) ([]byte, error) { + args := []string{"update-server-info"} + return g.gitCommand(dir, args...) +} + +func (g *GitHttp) gitCommand(dir string, args ...string) ([]byte, error) { + command := exec.Command(g.GitBinPath, args...) + command.Dir = dir + + return command.Output() +} diff --git a/pkg/go-git-http/pktparser.go b/pkg/go-git-http/pktparser.go new file mode 100644 index 0000000000000000000000000000000000000000..950804f6cc9d6bcb7458ec336c9197110be3ee64 --- /dev/null +++ b/pkg/go-git-http/pktparser.go @@ -0,0 +1,143 @@ +package githttp + +import ( + "encoding/hex" + "errors" + "fmt" +) + +// pktLineParser is a parser for git pkt-line Format, +// as documented in https://github.com/git/git/blob/master/Documentation/technical/protocol-common.txt. +// A zero value of pktLineParser is valid to use as a parser in ready state. +// Output should be read from Lines and Error after Step returns finished true. +// pktLineParser reads until a terminating "0000" flush-pkt. It's good for a single use only. +type pktLineParser struct { + // Lines contains all pkt-lines. + Lines []string + + // Error contains the first error encountered while parsing, or nil otherwise. + Error error + + // Internal state machine. + state state + next int // next is the number of bytes that need to be written to buf before its contents should be processed by the state machine. + buf []byte +} + +// Feed accumulates and parses data. +// It will return early if it reaches end of pkt-line data (indicated by a flush-pkt "0000"), +// or if it encounters a parsing error. +// It must not be called when state is done. +// When done, all of pkt-lines will be available in Lines, and Error will be set if any error occurred. +func (p *pktLineParser) Feed(data []byte) { + for { + // If not enough data to reach next state, append it to buf and return. + if len(data) < p.next { + p.buf = append(p.buf, data...) + p.next -= len(data) + return + } + + // There's enough data to reach next state. Take from data only what's needed. + b := data[:p.next] + data = data[p.next:] + p.buf = append(p.buf, b...) + p.next = 0 + + // Take a step to next state. + err := p.step() + if err != nil { + p.state = done + p.Error = err + return + } + + // Break out once reached done state. + if p.state == done { + return + } + } +} + +const ( + // pkt-len = 4*(HEXDIG) + pktLenSize = 4 +) + +type state uint8 + +const ( + ready state = iota + readingLen + readingPayload + done +) + +// step moves the state machine to the next state. +// buf must contain all the data ready for consumption for current state. +// It must not be called when state is done. +func (p *pktLineParser) step() error { + switch p.state { + case ready: + p.state = readingLen + p.next = pktLenSize + return nil + case readingLen: + // len(p.buf) is 4. + pktLen, err := parsePktLen(p.buf) + if err != nil { + return err + } + + switch { + case pktLen == 0: + p.state = done + p.next = 0 + p.buf = nil + return nil + default: + p.state = readingPayload + p.next = pktLen - pktLenSize // (pkt-len - 4)*(OCTET) + p.buf = p.buf[:0] + return nil + } + case readingPayload: + p.state = readingLen + p.next = pktLenSize + p.Lines = append(p.Lines, string(p.buf)) + p.buf = p.buf[:0] + return nil + default: + panic(fmt.Errorf("unreachable: %v", p.state)) + } +} + +// parsePktLen parses a pkt-len segment. +// len(b) must be 4. +func parsePktLen(b []byte) (int, error) { + pktLen, err := parseHex(b) + switch { + case err != nil: + return 0, err + case 1 <= pktLen && pktLen < pktLenSize: + return 0, fmt.Errorf("invalid pkt-len: %v", pktLen) + case pktLen > 65524: + // The maximum length of a pkt-line is 65524 bytes (65520 bytes of payload + 4 bytes of length data). + return 0, fmt.Errorf("invalid pkt-len: %v", pktLen) + } + return int(pktLen), nil +} + +// parseHex parses a 4-byte hex number. +// len(h) must be 4. +func parseHex(h []byte) (uint16, error) { + var b [2]uint8 + n, err := hex.Decode(b[:], h) + switch { + case err != nil: + return 0, err + case n != 2: + return 0, errors.New("short output") + } + return uint16(b[0])<<8 | uint16(b[1]), nil +} diff --git a/pkg/go-git-http/pktparser_test.go b/pkg/go-git-http/pktparser_test.go new file mode 100644 index 0000000000000000000000000000000000000000..f48c2f42bf08e1f63c1e5836c0be38b710236b15 --- /dev/null +++ b/pkg/go-git-http/pktparser_test.go @@ -0,0 +1,44 @@ +package githttp + +import ( + "encoding/hex" + "errors" + "reflect" + "testing" +) + +func TestParsePktLen(t *testing.T) { + tests := []struct { + in string + + wantLen int + wantErr error + }{ + // Valid pkt-len. + {"00a5", 165, nil}, + {"01a5", 421, nil}, + {"0032", 50, nil}, + {"000b", 11, nil}, + {"000B", 11, nil}, + + // Valud flush-pkt. + {"0000", 0, nil}, + + {"0001", 0, errors.New("invalid pkt-len: 1")}, + {"0003", 0, errors.New("invalid pkt-len: 3")}, + {"abyz", 0, hex.InvalidByteError('y')}, + {"-<%^", 0, hex.InvalidByteError('-')}, + + // Maximum length. + {"fff4", 65524, nil}, + {"fff5", 0, errors.New("invalid pkt-len: 65525")}, + {"ffff", 0, errors.New("invalid pkt-len: 65535")}, + } + + for _, tt := range tests { + gotLen, gotErr := parsePktLen([]byte(tt.in)) + if gotLen != tt.wantLen || !reflect.DeepEqual(gotErr, tt.wantErr) { + t.Errorf("test %q:\n got: %#v, %#v\nwant: %#v, %#v\n", tt.in, gotLen, gotErr, tt.wantLen, tt.wantErr) + } + } +} diff --git a/pkg/go-git-http/routing.go b/pkg/go-git-http/routing.go new file mode 100644 index 0000000000000000000000000000000000000000..fc64f7200cef474adf05afd8c9cd0056541ad90b --- /dev/null +++ b/pkg/go-git-http/routing.go @@ -0,0 +1,120 @@ +package githttp + +import ( + "fmt" + "net/http" + "os" + "regexp" + "strings" +) + +type Service struct { + Method string + Handler func(HandlerReq) error + Rpc string +} + +type HandlerReq struct { + w http.ResponseWriter + r *http.Request + Rpc string + Dir string + File string +} + +// Routing regexes +var ( + _serviceRpcUpload = regexp.MustCompile("git/(.*?)/git-upload-pack$") + _serviceRpcReceive = regexp.MustCompile("git/(.*?)/git-receive-pack$") + _getInfoRefs = regexp.MustCompile("git/(.*?)/info/refs$") + _getHead = regexp.MustCompile("git/(.*?)/HEAD$") + _getAlternates = regexp.MustCompile("git/(.*?)/objects/info/alternates$") + _getHttpAlternates = regexp.MustCompile("git/(.*?)/objects/info/http-alternates$") + _getInfoPacks = regexp.MustCompile("git/(.*?)/objects/info/packs$") + _getInfoFile = regexp.MustCompile("git/(.*?)/objects/info/[^/]*$") + _getLooseObject = regexp.MustCompile("git/(.*?)/objects/[0-9a-f]{2}/[0-9a-f]{38}$") + _getPackFile = regexp.MustCompile("git/(.*?)/objects/pack/pack-[0-9a-f]{40}\\.pack$") + _getIdxFile = regexp.MustCompile("git/(.*?)/objects/pack/pack-[0-9a-f]{40}\\.idx$") +) + +func (g *GitHttp) services() map[*regexp.Regexp]Service { + return map[*regexp.Regexp]Service{ + _serviceRpcUpload: Service{"POST", g.serviceRpc, "upload-pack"}, + _serviceRpcReceive: Service{"POST", g.serviceRpc, "receive-pack"}, + _getInfoRefs: Service{"GET", g.getInfoRefs, ""}, + _getHead: Service{"GET", g.getTextFile, ""}, + _getAlternates: Service{"GET", g.getTextFile, ""}, + _getHttpAlternates: Service{"GET", g.getTextFile, ""}, + _getInfoPacks: Service{"GET", g.getInfoPacks, ""}, + _getInfoFile: Service{"GET", g.getTextFile, ""}, + _getLooseObject: Service{"GET", g.getLooseObject, ""}, + _getPackFile: Service{"GET", g.getPackFile, ""}, + _getIdxFile: Service{"GET", g.getIdxFile, ""}, + } +} + +// getService return's the service corresponding to the +// current http.Request's URL +// as well as the name of the repo +func (g *GitHttp) getService(path string) (string, *Service) { + for re, service := range g.services() { + if m := re.FindStringSubmatch(path); m != nil { + return m[1], &service + } + } + + // No match + return "", nil +} + +// Request handling function +func (g *GitHttp) requestHandler(w http.ResponseWriter, r *http.Request) { + // Get service for URL + repo, service := g.getService(r.URL.Path) + + fmt.Println("git handler", r.URL.Path, repo) + + // No url match + if service == nil { + renderNotFound(w) + return + } + + // Bad method + if service.Method != r.Method { + renderMethodNotAllowed(w, r) + return + } + + // Rpc type + rpc := service.Rpc + + // Get specific file + file := strings.Replace(r.URL.Path, repo+"/", "", 1) + + // Resolve directory + dir, err := g.getGitDir(repo) + + // Repo not found on disk + if err != nil { + renderNotFound(w) + return + } + + // Build request info for handler + hr := HandlerReq{w, r, rpc, dir, file} + + // Call handler + if err := service.Handler(hr); err != nil { + if os.IsNotExist(err) { + renderNotFound(w) + return + } + switch err.(type) { + case *ErrorNoAccess: + renderNoAccess(w) + return + } + http.Error(w, err.Error(), 500) + } +} diff --git a/pkg/go-git-http/rpc_reader.go b/pkg/go-git-http/rpc_reader.go new file mode 100644 index 0000000000000000000000000000000000000000..cb89db85b9126f5ab5cc3d82fa0036e662a0b582 --- /dev/null +++ b/pkg/go-git-http/rpc_reader.go @@ -0,0 +1,120 @@ +package githttp + +import ( + "io" + "regexp" + "strings" +) + +// RpcReader scans for events in the incoming rpc request data. +type RpcReader struct { + // Underlying reader (to relay calls to). + io.Reader + + // Rpc type (receive-pack or upload-pack). + Rpc string + + // List of events RpcReader has picked up through scanning. + // These events do not have the Dir field set. + Events []Event + + pktLineParser pktLineParser +} + +// Read implements the io.Reader interface. +func (r *RpcReader) Read(p []byte) (n int, err error) { + // Relay call + n, err = r.Reader.Read(p) + + // Scan for events + if n > 0 { + r.scan(p[:n]) + } + + return n, err +} + +func (r *RpcReader) scan(data []byte) { + if r.pktLineParser.state == done { + return + } + + r.pktLineParser.Feed(data) + + // If parsing has just finished, process its output once. + if r.pktLineParser.state == done { + if r.pktLineParser.Error != nil { + return + } + + // When we get here, we're done collecting all pkt-lines successfully + // and can now extract relevant events. + switch r.Rpc { + case "receive-pack": + for _, line := range r.pktLineParser.Lines { + events := scanPush(line) + r.Events = append(r.Events, events...) + } + case "upload-pack": + total := strings.Join(r.pktLineParser.Lines, "") + events := scanFetch(total) + r.Events = append(r.Events, events...) + } + } +} + +// TODO: Avoid using regexp to parse a well documented binary protocol with an open source +// implementation. There should not be a need for regexp. + +// receivePackRegex is used once per pkt-line. +var receivePackRegex = regexp.MustCompile("([0-9a-fA-F]{40}) ([0-9a-fA-F]{40}) refs\\/(heads|tags)\\/(.+?)(\x00|$)") + +func scanPush(line string) []Event { + matches := receivePackRegex.FindAllStringSubmatch(line, -1) + + if matches == nil { + return nil + } + + var events []Event + for _, m := range matches { + e := Event{ + Last: m[1], + Commit: m[2], + } + + // Handle pushes to branches and tags differently + if m[3] == "heads" { + e.Type = PUSH + e.Branch = m[4] + } else { + e.Type = TAG + e.Tag = m[4] + } + + events = append(events, e) + } + + return events +} + +// uploadPackRegex is used once on the entire header data. +var uploadPackRegex = regexp.MustCompile(`^want ([0-9a-fA-F]{40})`) + +func scanFetch(total string) []Event { + matches := uploadPackRegex.FindAllStringSubmatch(total, -1) + + if matches == nil { + return nil + } + + var events []Event + for _, m := range matches { + events = append(events, Event{ + Type: FETCH, + Commit: m[1], + }) + } + + return events +} diff --git a/pkg/go-git-http/rpc_reader_test.go b/pkg/go-git-http/rpc_reader_test.go new file mode 100644 index 0000000000000000000000000000000000000000..13a17cc560c40e3d196f63a23233a7b6eea5480a --- /dev/null +++ b/pkg/go-git-http/rpc_reader_test.go @@ -0,0 +1,193 @@ +package githttp_test + +import ( + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/AaronO/go-git-http" +) + +func TestRpcReader(t *testing.T) { + tests := []struct { + rpc string + file string + + want []githttp.Event + }{ + { + rpc: "receive-pack", + file: "receive-pack.0", + + want: []githttp.Event{ + (githttp.Event)(githttp.Event{ + Type: (githttp.EventType)(githttp.PUSH), + Commit: (string)("92eef6dcb9cc198bc3ac6010c108fa482773f116"), + Dir: (string)(""), + Tag: (string)(""), + Last: (string)("0000000000000000000000000000000000000000"), + Branch: (string)("master"), + Error: (error)(nil), + Request: (*http.Request)(nil), + }), + }, + }, + + // A tag using letters only. + { + rpc: "receive-pack", + file: "receive-pack.1", + + want: []githttp.Event{ + (githttp.Event)(githttp.Event{ + Type: (githttp.EventType)(githttp.TAG), + Commit: (string)("3da295397738f395c2ca5fd5570f01a9fcea3be3"), + Dir: (string)(""), + Tag: (string)("sometextualtag"), + Last: (string)("0000000000000000000000000000000000000000"), + Branch: (string)(""), + Error: (error)(nil), + Request: (*http.Request)(nil), + }), + }, + }, + + // A tag containing the string "00". + { + rpc: "receive-pack", + file: "receive-pack.2", + + want: []githttp.Event{ + (githttp.Event)(githttp.Event{ + Type: (githttp.EventType)(githttp.TAG), + Commit: (string)("3da295397738f395c2ca5fd5570f01a9fcea3be3"), + Dir: (string)(""), + Tag: (string)("1.000.1"), + Last: (string)("0000000000000000000000000000000000000000"), + Branch: (string)(""), + Error: (error)(nil), + Request: (*http.Request)(nil), + }), + }, + }, + + // Multiple tags containing string "00" in one git push operation. + { + rpc: "receive-pack", + file: "receive-pack.3", + + want: []githttp.Event{ + (githttp.Event)(githttp.Event{ + Type: (githttp.EventType)(githttp.TAG), + Commit: (string)("3da295397738f395c2ca5fd5570f01a9fcea3be3"), + Dir: (string)(""), + Tag: (string)("1.000.2"), + Last: (string)("0000000000000000000000000000000000000000"), + Branch: (string)(""), + Error: (error)(nil), + Request: (*http.Request)(nil), + }), + (githttp.Event)(githttp.Event{ + Type: (githttp.EventType)(githttp.TAG), + Commit: (string)("3da295397738f395c2ca5fd5570f01a9fcea3be3"), + Dir: (string)(""), + Tag: (string)("1.000.3"), + Last: (string)("0000000000000000000000000000000000000000"), + Branch: (string)(""), + Error: (error)(nil), + Request: (*http.Request)(nil), + }), + (githttp.Event)(githttp.Event{ + Type: (githttp.EventType)(githttp.TAG), + Commit: (string)("3da295397738f395c2ca5fd5570f01a9fcea3be3"), + Dir: (string)(""), + Tag: (string)("1.000.4"), + Last: (string)("0000000000000000000000000000000000000000"), + Branch: (string)(""), + Error: (error)(nil), + Request: (*http.Request)(nil), + }), + }, + }, + + { + rpc: "upload-pack", + file: "upload-pack.0", + + want: []githttp.Event{ + (githttp.Event)(githttp.Event{ + Type: (githttp.EventType)(githttp.FETCH), + Commit: (string)("a647ec2ea40ee9ca35d32232dc28de22b1537e00"), + Dir: (string)(""), + Tag: (string)(""), + Last: (string)(""), + Branch: (string)(""), + Error: (error)(nil), + Request: (*http.Request)(nil), + }), + }, + }, + + { + rpc: "upload-pack", + file: "upload-pack.1", + + want: []githttp.Event{ + (githttp.Event)(githttp.Event{ + Type: (githttp.EventType)(githttp.FETCH), + Commit: (string)("92eef6dcb9cc198bc3ac6010c108fa482773f116"), + Dir: (string)(""), + Tag: (string)(""), + Last: (string)(""), + Branch: (string)(""), + Error: (error)(nil), + Request: (*http.Request)(nil), + }), + }, + }, + } + + for _, tt := range tests { + f, err := os.Open(filepath.Join("testdata", tt.file)) + if err != nil { + t.Fatal(err) + } + + r := fragmentedReader{f} + + rr := &githttp.RpcReader{ + Reader: r, + Rpc: tt.rpc, + } + + _, err = io.Copy(ioutil.Discard, rr) + if err != nil { + t.Errorf("io.Copy: %v", err) + } + + f.Close() + + if got := rr.Events; !reflect.DeepEqual(got, tt.want) { + t.Errorf("test %q/%q:\n got: %#v\nwant: %#v\n", tt.rpc, tt.file, got, tt.want) + } + } +} + +// fragmentedReader reads from R, with each Read call returning at most fragmentLen bytes even +// if len(p) is greater than fragmentLen. +// It purposefully adds a layer of inefficiency around R, and exists for testing purposes only. +type fragmentedReader struct { + R io.Reader // Underlying reader. +} + +func (r fragmentedReader) Read(p []byte) (n int, err error) { + const fragmentLen = 1 + if len(p) <= fragmentLen { + return r.R.Read(p) + } + return r.R.Read(p[:fragmentLen]) +} diff --git a/pkg/go-git-http/testdata/receive-pack.0 b/pkg/go-git-http/testdata/receive-pack.0 new file mode 100644 index 0000000000000000000000000000000000000000..96cb11616dd2542afbffc5f3680576abdb9df5ac Binary files /dev/null and b/pkg/go-git-http/testdata/receive-pack.0 differ diff --git a/pkg/go-git-http/testdata/receive-pack.1 b/pkg/go-git-http/testdata/receive-pack.1 new file mode 100644 index 0000000000000000000000000000000000000000..fdfe67be413661267d3dac802774dda0a249524c Binary files /dev/null and b/pkg/go-git-http/testdata/receive-pack.1 differ diff --git a/pkg/go-git-http/testdata/receive-pack.2 b/pkg/go-git-http/testdata/receive-pack.2 new file mode 100644 index 0000000000000000000000000000000000000000..05447175294f62321148675f9c0b7ba6f1428268 Binary files /dev/null and b/pkg/go-git-http/testdata/receive-pack.2 differ diff --git a/pkg/go-git-http/testdata/receive-pack.3 b/pkg/go-git-http/testdata/receive-pack.3 new file mode 100644 index 0000000000000000000000000000000000000000..5ea98cd56615908443d1927cae46aa1374dda9a5 Binary files /dev/null and b/pkg/go-git-http/testdata/receive-pack.3 differ diff --git a/pkg/go-git-http/testdata/upload-pack.0 b/pkg/go-git-http/testdata/upload-pack.0 new file mode 100644 index 0000000000000000000000000000000000000000..9269a330413daf1fe038d31cbaa29cf8e7bc5836 --- /dev/null +++ b/pkg/go-git-http/testdata/upload-pack.0 @@ -0,0 +1,34 @@ +0092want a647ec2ea40ee9ca35d32232dc28de22b1537e00 multi_ack_detailed side-band-64k thin-pack include-tag ofs-delta agent=git/2.5.4.(Apple.Git-61) +00000032have 92eef6dcb9cc198bc3ac6010c108fa482773f116 +0032have 0e2a2938f51c984c4f6fe68400a77b2748e3aac7 +0032have 9b40a23d8105a4d8f5babf9756742b353bc0bf86 +0032have 6dcded2bb74142bd97b346c70e34a7a117dc359d +0032have 5e247562e60140946b2996ab428ea4ebfd8f7aef +0032have e399f2b29d0efe66a9597239abdbec5e2960e172 +0032have 6779fd7460bf02ffa67fd7c6fa412fedc22eea02 +0032have 30d62cac4ee3185b5670e9ca366a409a2dade471 +0032have 088a47a7b0d7141b71dbf0fabfbad66c61ffb99f +0032have 2c960968453207a2a66309e2e752759888345900 +0032have 4f0b8f6c5df1a903b204cdc9ff20a7e00873d73d +0032have d403d8b126c2d566eb105102972df356f7824406 +0032have 926801b90aa8180679eccb0ee3231de10903df9b +0032have accbc2b1a251e2cd6dd0c3fba74c2f7789a5addd +0032have f6eb2af801c0722b3112346da3d1fd68e164cb74 +0032have 5e3817ddb991f9530c9cfc7c4a5cf5203257fbd8 +00000032have 5ee7e39b927366c74181b199e0da78467699dd3d +0032have 5d1d1ba532f180d55a6fc7b23bda8162693447d7 +0032have 5dc05dfd1827333036a03b455267432157d8315a +0032have 3378b0e3808ce82185bdf57d97c5d5c7655f14d3 +0032have 1aa139246412abf51c7f6402deb3d9ee84cfba6b +0032have e94b8b4c1c9103d4aa8656f55c54ce91f4941237 +0032have 2edfc6f95708194e23c92bf80da115fd76953cdc +0032have 64b4dcae2edfdd6201efe622f6394694281c1fab +0032have 6224112203b656464ee55f42eca4b80a9d8ae854 +0032have 230517d50257e5f8d2706976c7ed7d333e2b9916 +0032have 6b8d66508e23b76ecf8236b45218c77d2e66c7df +0032have 622958e856b24f771218aad8d26264d403df0021 +0032have 1c82218b4749a5eec2750b876a7544105d357db5 +0032have 861ceef44614479fecb6c5118773afc73c22fc31 +0032have 74b86980e2e8e3d47b58a719e854819cab1ffb8b +0032have bb5f1a2dbd16acb79584beb4021053c3718b07ce +00000009done diff --git a/pkg/go-git-http/testdata/upload-pack.1 b/pkg/go-git-http/testdata/upload-pack.1 new file mode 100644 index 0000000000000000000000000000000000000000..38c09e586aa2314d86761c29266bb85a8cb82d09 --- /dev/null +++ b/pkg/go-git-http/testdata/upload-pack.1 @@ -0,0 +1,3 @@ +0086want 92eef6dcb9cc198bc3ac6010c108fa482773f116 multi_ack_detailed side-band-64k thin-pack ofs-delta agent=git/2.5.4.(Apple.Git-61) +0032want 92eef6dcb9cc198bc3ac6010c108fa482773f116 +00000009done diff --git a/pkg/go-git-http/utils.go b/pkg/go-git-http/utils.go new file mode 100644 index 0000000000000000000000000000000000000000..545e9e37d307e805a832fefae5c2a30231211cb9 --- /dev/null +++ b/pkg/go-git-http/utils.go @@ -0,0 +1,93 @@ +package githttp + +import ( + "compress/flate" + "compress/gzip" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" +) + +// requestReader returns an io.ReadCloser +// that will decode data if needed, depending on the +// "content-encoding" header +func requestReader(req *http.Request) (io.ReadCloser, error) { + switch req.Header.Get("content-encoding") { + case "gzip": + return gzip.NewReader(req.Body) + case "deflate": + return flate.NewReader(req.Body), nil + } + + // If no encoding, use raw body + return req.Body, nil +} + +// HTTP parsing utility functions + +func getServiceType(r *http.Request) string { + service_type := r.FormValue("service") + + if s := strings.HasPrefix(service_type, "git-"); !s { + return "" + } + + return strings.Replace(service_type, "git-", "", 1) +} + +// HTTP error response handling functions + +func renderMethodNotAllowed(w http.ResponseWriter, r *http.Request) { + if r.Proto == "HTTP/1.1" { + w.WriteHeader(http.StatusMethodNotAllowed) + w.Write([]byte("Method Not Allowed")) + } else { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte("Bad Request")) + } +} + +func renderNotFound(w http.ResponseWriter) { + w.WriteHeader(http.StatusNotFound) + w.Write([]byte("Not Found")) +} + +func renderNoAccess(w http.ResponseWriter) { + w.WriteHeader(http.StatusForbidden) + w.Write([]byte("Forbidden")) +} + +// Packet-line handling function + +func packetFlush() []byte { + return []byte("0000") +} + +func packetWrite(str string) []byte { + s := strconv.FormatInt(int64(len(str)+4), 16) + + if len(s)%4 != 0 { + s = strings.Repeat("0", 4-len(s)%4) + s + } + + return []byte(s + str) +} + +// Header writing functions + +func hdrNocache(w http.ResponseWriter) { + w.Header().Set("Expires", "Fri, 01 Jan 1980 00:00:00 GMT") + w.Header().Set("Pragma", "no-cache") + w.Header().Set("Cache-Control", "no-cache, max-age=0, must-revalidate") +} + +func hdrCacheForever(w http.ResponseWriter) { + now := time.Now().Unix() + expires := now + 31536000 + w.Header().Set("Date", fmt.Sprintf("%d", now)) + w.Header().Set("Expires", fmt.Sprintf("%d", expires)) + w.Header().Set("Cache-Control", "public, max-age=31536000") +} diff --git a/pkg/go-git-http/version.go b/pkg/go-git-http/version.go new file mode 100644 index 0000000000000000000000000000000000000000..2f6acc340f2db53f98d71065427c606c1810b7cc --- /dev/null +++ b/pkg/go-git-http/version.go @@ -0,0 +1,3 @@ +package githttp + +const VERSION = "1.0.0" diff --git a/pkg/smithy/smithy.go b/pkg/smithy/smithy.go index 3cba1f6c63bbb9789deb9774bba2c9ba80706433..e9e8dee164e1c919f9bcef6bc74c4ee573f16ae2 100644 --- a/pkg/smithy/smithy.go +++ b/pkg/smithy/smithy.go @@ -42,6 +42,8 @@ "github.com/go-git/go-git/v5/plumbing/storer" "github.com/rakyll/statik/fs" "github.com/yuin/goldmark" + "github.com/honza/smithy/pkg/go-git-http" + _ "github.com/honza/smithy/pkg/statik" ) @@ -279,9 +281,17 @@ "Branches": bs, "Tags": ts, "Readme": template.HTML(formattedReadme), "Repo": repo, + + "Host": smithyConfig.Host, }) } +func RepoGitView(ctx *gin.Context, urlParts []string) { + smithyConfig := ctx.MustGet("config").(SmithyConfig) + git := githttp.New(smithyConfig.Git.Root) + git.ServeHTTP(ctx.Writer, ctx.Request) +} + func RefsView(ctx *gin.Context, urlParts []string) { repoName := urlParts[0] smithyConfig := ctx.MustGet("config").(SmithyConfig) @@ -673,6 +683,7 @@ // A filepath is a list of labels label := `[a-zA-Z0-9\-~\.]+` indexUrl := regexp.MustCompile(`^/$`) + repoGitUrl := regexp.MustCompile(`^/git/(?P<repo>` + label + `)`) repoIndexUrl := regexp.MustCompile(`^/(?P<repo>` + label + `)$`) refsUrl := regexp.MustCompile(`^/(?P<repo>` + label + `)/refs$`) logDefaultUrl := regexp.MustCompile(`^/(?P<repo>` + label + `)/log$`) @@ -686,6 +697,7 @@ return []Route{ {Pattern: indexUrl, View: IndexView}, {Pattern: repoIndexUrl, View: RepoIndexView}, + {Pattern: repoGitUrl, View: RepoGitView}, {Pattern: refsUrl, View: RefsView}, {Pattern: logDefaultUrl, View: LogViewDefault}, {Pattern: logUrl, View: LogView}, @@ -832,7 +844,7 @@ fileSystemHandler := InitFileSystemHandler(config) routes := CompileRoutes() - router.GET("*path", func(ctx *gin.Context) { + router.Any("*path", func(ctx *gin.Context) { Dispatch(ctx, routes, fileSystemHandler) })