From 74f1ff6bd0b4402c55dd1a6309343e3ff00c68b9 Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Sun, 6 Nov 2016 16:31:59 +0100 Subject: [PATCH 01/37] Import github.com/git-lfs/lfs-test-server as lfs module base Imported commit is 3968aac269a77b73924649b9412ae03f7ccd3198 Removed: Dockerfile CONTRIBUTING.md mgmt* script/ vendor/ kvlogger.go .dockerignore .gitignore README.md --- modules/lfs/LICENSE | 19 ++ modules/lfs/config.go | 67 +++++ modules/lfs/content_store.go | 102 ++++++++ modules/lfs/content_store_test.go | 173 +++++++++++++ modules/lfs/main.go | 112 +++++++++ modules/lfs/meta_store.go | 303 ++++++++++++++++++++++ modules/lfs/meta_store_test.go | 111 ++++++++ modules/lfs/server.go | 400 +++++++++++++++++++++++++++++ modules/lfs/server_test.go | 406 ++++++++++++++++++++++++++++++ modules/lfs/tracking_listener.go | 95 +++++++ 10 files changed, 1788 insertions(+) create mode 100644 modules/lfs/LICENSE create mode 100644 modules/lfs/config.go create mode 100644 modules/lfs/content_store.go create mode 100644 modules/lfs/content_store_test.go create mode 100644 modules/lfs/main.go create mode 100644 modules/lfs/meta_store.go create mode 100644 modules/lfs/meta_store_test.go create mode 100644 modules/lfs/server.go create mode 100644 modules/lfs/server_test.go create mode 100644 modules/lfs/tracking_listener.go diff --git a/modules/lfs/LICENSE b/modules/lfs/LICENSE new file mode 100644 index 0000000000000..3d7c5a5c51101 --- /dev/null +++ b/modules/lfs/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) GitHub, Inc. and LFS Test Server contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/modules/lfs/config.go b/modules/lfs/config.go new file mode 100644 index 0000000000000..ee4bdff4d83a0 --- /dev/null +++ b/modules/lfs/config.go @@ -0,0 +1,67 @@ +package main + +import ( + "fmt" + "os" + "reflect" + "strings" +) + +// Configuration holds application configuration. Values will be pulled from +// environment variables, prefixed by keyPrefix. Default values can be added +// via tags. +type Configuration struct { + Listen string `config:"tcp://:8080"` + Host string `config:"localhost:8080"` + MetaDB string `config:"lfs.db"` + ContentPath string `config:"lfs-content"` + AdminUser string `config:""` + AdminPass string `config:""` + Cert string `config:""` + Key string `config:""` + Scheme string `config:"http"` + Public string `config:"public"` +} + +func (c *Configuration) IsHTTPS() bool { + return strings.Contains(Config.Scheme, "https") +} + +func (c *Configuration) IsPublic() bool { + switch Config.Public { + case "1", "true", "TRUE": + return true + } + return false +} + +// Config is the global app configuration +var Config = &Configuration{} + +const keyPrefix = "LFS" + +func init() { + te := reflect.TypeOf(Config).Elem() + ve := reflect.ValueOf(Config).Elem() + + for i := 0; i < te.NumField(); i++ { + sf := te.Field(i) + name := sf.Name + field := ve.FieldByName(name) + + envVar := strings.ToUpper(fmt.Sprintf("%s_%s", keyPrefix, name)) + env := os.Getenv(envVar) + tag := sf.Tag.Get("config") + + if env == "" && tag != "" { + env = tag + } + + field.SetString(env) + } + + if port := os.Getenv("PORT"); port != "" { + // If $PORT is set, override LFS_LISTEN. This is useful for deploying to Heroku. + Config.Listen = "tcp://:" + port + } +} diff --git a/modules/lfs/content_store.go b/modules/lfs/content_store.go new file mode 100644 index 0000000000000..aff6e87b5476f --- /dev/null +++ b/modules/lfs/content_store.go @@ -0,0 +1,102 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "os" + "path/filepath" +) + +var ( + errHashMismatch = errors.New("Content hash does not match OID") + errSizeMismatch = errors.New("Content size does not match") +) + +// ContentStore provides a simple file system based storage. +type ContentStore struct { + basePath string +} + +// NewContentStore creates a ContentStore at the base directory. +func NewContentStore(base string) (*ContentStore, error) { + if err := os.MkdirAll(base, 0750); err != nil { + return nil, err + } + + return &ContentStore{base}, nil +} + +// Get takes a Meta object and retreives the content from the store, returning +// it as an io.Reader. If fromByte > 0, the reader starts from that byte +func (s *ContentStore) Get(meta *MetaObject, fromByte int64) (io.Reader, error) { + path := filepath.Join(s.basePath, transformKey(meta.Oid)) + + f, err := os.Open(path) + if err != nil { + return nil, err + } + if fromByte > 0 { + _, err = f.Seek(fromByte, os.SEEK_CUR) + } + return f, err +} + +// Put takes a Meta object and an io.Reader and writes the content to the store. +func (s *ContentStore) Put(meta *MetaObject, r io.Reader) error { + path := filepath.Join(s.basePath, transformKey(meta.Oid)) + tmpPath := path + ".tmp" + + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0750); err != nil { + return err + } + + file, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0640) + if err != nil { + return err + } + defer os.Remove(tmpPath) + + hash := sha256.New() + hw := io.MultiWriter(hash, file) + + written, err := io.Copy(hw, r) + if err != nil { + file.Close() + return err + } + file.Close() + + if written != meta.Size { + return errSizeMismatch + } + + shaStr := hex.EncodeToString(hash.Sum(nil)) + if shaStr != meta.Oid { + return errHashMismatch + } + + if err := os.Rename(tmpPath, path); err != nil { + return err + } + return nil +} + +// Exists returns true if the object exists in the content store. +func (s *ContentStore) Exists(meta *MetaObject) bool { + path := filepath.Join(s.basePath, transformKey(meta.Oid)) + if _, err := os.Stat(path); os.IsNotExist(err) { + return false + } + return true +} + +func transformKey(key string) string { + if len(key) < 5 { + return key + } + + return filepath.Join(key[0:2], key[2:4], key[4:len(key)]) +} diff --git a/modules/lfs/content_store_test.go b/modules/lfs/content_store_test.go new file mode 100644 index 0000000000000..9c196489e30c8 --- /dev/null +++ b/modules/lfs/content_store_test.go @@ -0,0 +1,173 @@ +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "testing" +) + +var contentStore *ContentStore + +func TestContenStorePut(t *testing.T) { + setup() + defer teardown() + + m := &MetaObject{ + Oid: "6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72", + Size: 12, + } + + b := bytes.NewBuffer([]byte("test content")) + + if err := contentStore.Put(m, b); err != nil { + t.Fatalf("expected put to succeed, got: %s", err) + } + + path := "content-store-test/6a/e8/a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72" + if _, err := os.Stat(path); os.IsNotExist(err) { + t.Fatalf("expected content to exist after putting") + } +} + +func TestContenStorePutHashMismatch(t *testing.T) { + setup() + defer teardown() + + m := &MetaObject{ + Oid: "6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72", + Size: 12, + } + + b := bytes.NewBuffer([]byte("bogus content")) + + if err := contentStore.Put(m, b); err == nil { + t.Fatal("expected put with bogus content to fail") + } + + path := "content-store-test/6a/e8/a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72" + if _, err := os.Stat(path); err == nil { + t.Fatalf("expected content to not exist after putting bogus content") + } +} + +func TestContenStorePutSizeMismatch(t *testing.T) { + setup() + defer teardown() + + m := &MetaObject{ + Oid: "6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72", + Size: 14, + } + + b := bytes.NewBuffer([]byte("test content")) + + if err := contentStore.Put(m, b); err == nil { + t.Fatal("expected put with bogus size to fail") + } + + path := "content-store-test/6a/e8/a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72" + if _, err := os.Stat(path); err == nil { + t.Fatalf("expected content to not exist after putting bogus size") + } +} + +func TestContenStoreGet(t *testing.T) { + setup() + defer teardown() + + m := &MetaObject{ + Oid: "6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72", + Size: 12, + } + + b := bytes.NewBuffer([]byte("test content")) + + if err := contentStore.Put(m, b); err != nil { + t.Fatalf("expected put to succeed, got: %s", err) + } + + r, err := contentStore.Get(m, 0) + if err != nil { + t.Fatalf("expected get to succeed, got: %s", err) + } + + by, _ := ioutil.ReadAll(r) + if string(by) != "test content" { + t.Fatalf("expected to read content, got: %s", string(by)) + } +} + +func TestContenStoreGetWithRange(t *testing.T) { + setup() + defer teardown() + + m := &MetaObject{ + Oid: "6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72", + Size: 12, + } + + b := bytes.NewBuffer([]byte("test content")) + + if err := contentStore.Put(m, b); err != nil { + t.Fatalf("expected put to succeed, got: %s", err) + } + + r, err := contentStore.Get(m, 5) + if err != nil { + t.Fatalf("expected get to succeed, got: %s", err) + } + + by, _ := ioutil.ReadAll(r) + if string(by) != "content" { + t.Fatalf("expected to read content, got: %s", string(by)) + } +} + +func TestContenStoreGetNonExisting(t *testing.T) { + setup() + defer teardown() + + _, err := contentStore.Get(&MetaObject{Oid: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, 0) + if err == nil { + t.Fatalf("expected to get an error, but content existed") + } +} + +func TestContenStoreExists(t *testing.T) { + setup() + defer teardown() + + m := &MetaObject{ + Oid: "6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72", + Size: 12, + } + + b := bytes.NewBuffer([]byte("test content")) + + if contentStore.Exists(m) { + t.Fatalf("expected content to not exist yet") + } + + if err := contentStore.Put(m, b); err != nil { + t.Fatalf("expected put to succeed, got: %s", err) + } + + if !contentStore.Exists(m) { + t.Fatalf("expected content to exist") + } +} + +func setup() { + store, err := NewContentStore("content-store-test") + if err != nil { + fmt.Printf("error initializing content store: %s\n", err) + os.Exit(1) + } + contentStore = store +} + +func teardown() { + os.RemoveAll("content-store-test") +} diff --git a/modules/lfs/main.go b/modules/lfs/main.go new file mode 100644 index 0000000000000..f6069c4d4f05d --- /dev/null +++ b/modules/lfs/main.go @@ -0,0 +1,112 @@ +package main + +import ( + "crypto/tls" + "fmt" + "net" + "os" + "os/signal" + "syscall" + "time" +) + +const ( + contentMediaType = "application/vnd.git-lfs" + metaMediaType = contentMediaType + "+json" + version = "0.3.0" +) + +var ( + logger = NewKVLogger(os.Stdout) +) + +// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted +// connections. It's used by ListenAndServe and ListenAndServeTLS so +// dead TCP connections (e.g. closing laptop mid-download) eventually +// go away. +type tcpKeepAliveListener struct { + *net.TCPListener +} + +func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) { + tc, err := ln.AcceptTCP() + if err != nil { + return + } + tc.SetKeepAlive(true) + tc.SetKeepAlivePeriod(3 * time.Minute) + return tc, nil +} + +func wrapHttps(l net.Listener, cert, key string) (net.Listener, error) { + var err error + + config := &tls.Config{} + + if config.NextProtos == nil { + config.NextProtos = []string{"http/1.1"} + } + + config.Certificates = make([]tls.Certificate, 1) + config.Certificates[0], err = tls.LoadX509KeyPair(cert, key) + if err != nil { + return nil, err + } + + netListener := l.(*TrackingListener).Listener + + tlsListener := tls.NewListener(tcpKeepAliveListener{netListener.(*net.TCPListener)}, config) + return tlsListener, nil +} + +func main() { + if len(os.Args) == 2 && os.Args[1] == "-v" { + fmt.Println(version) + os.Exit(0) + } + + var listener net.Listener + + tl, err := NewTrackingListener(Config.Listen) + if err != nil { + logger.Fatal(kv{"fn": "main", "err": "Could not create listener: " + err.Error()}) + } + + listener = tl + + if Config.IsHTTPS() { + logger.Log(kv{"fn": "main", "msg": "Using https"}) + listener, err = wrapHttps(tl, Config.Cert, Config.Key) + if err != nil { + logger.Fatal(kv{"fn": "main", "err": "Could not create https listener: " + err.Error()}) + } + } + + metaStore, err := NewMetaStore(Config.MetaDB) + if err != nil { + logger.Fatal(kv{"fn": "main", "err": "Could not open the meta store: " + err.Error()}) + } + + contentStore, err := NewContentStore(Config.ContentPath) + if err != nil { + logger.Fatal(kv{"fn": "main", "err": "Could not open the content store: " + err.Error()}) + } + + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGHUP) + go func(c chan os.Signal, listener net.Listener) { + for { + sig := <-c + switch sig { + case syscall.SIGHUP: // Graceful shutdown + tl.Close() + } + } + }(c, tl) + + logger.Log(kv{"fn": "main", "msg": "listening", "pid": os.Getpid(), "addr": Config.Listen, "version": version}) + + app := NewApp(contentStore, metaStore) + app.Serve(listener) + tl.WaitForChildren() +} diff --git a/modules/lfs/meta_store.go b/modules/lfs/meta_store.go new file mode 100644 index 0000000000000..e189ded2e48a6 --- /dev/null +++ b/modules/lfs/meta_store.go @@ -0,0 +1,303 @@ +package main + +import ( + "bytes" + "encoding/base64" + "encoding/gob" + "errors" + "strings" + "time" + + "github.com/boltdb/bolt" +) + +// MetaStore implements a metadata storage. It stores user credentials and Meta information +// for objects. The storage is handled by boltdb. +type MetaStore struct { + db *bolt.DB +} + +var ( + errNoBucket = errors.New("Bucket not found") + errObjectNotFound = errors.New("Object not found") +) + +var ( + usersBucket = []byte("users") + objectsBucket = []byte("objects") +) + +// NewMetaStore creates a new MetaStore using the boltdb database at dbFile. +func NewMetaStore(dbFile string) (*MetaStore, error) { + db, err := bolt.Open(dbFile, 0600, &bolt.Options{Timeout: 1 * time.Second}) + if err != nil { + return nil, err + } + + db.Update(func(tx *bolt.Tx) error { + if _, err := tx.CreateBucketIfNotExists(usersBucket); err != nil { + return err + } + + if _, err := tx.CreateBucketIfNotExists(objectsBucket); err != nil { + return err + } + + return nil + }) + + return &MetaStore{db: db}, nil +} + +// Get retrieves the Meta information for an object given information in +// RequestVars +func (s *MetaStore) Get(v *RequestVars) (*MetaObject, error) { + if !s.authenticate(v.Authorization) { + return nil, newAuthError() + } + meta, error := s.UnsafeGet(v) + return meta, error +} + +// Get retrieves the Meta information for an object given information in +// RequestVars +// DO NOT CHECK authentication, as it is supposed to have been done before +func (s *MetaStore) UnsafeGet(v *RequestVars) (*MetaObject, error) { + var meta MetaObject + + err := s.db.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket(objectsBucket) + if bucket == nil { + return errNoBucket + } + + value := bucket.Get([]byte(v.Oid)) + if len(value) == 0 { + return errObjectNotFound + } + + dec := gob.NewDecoder(bytes.NewBuffer(value)) + return dec.Decode(&meta) + }) + + if err != nil { + return nil, err + } + + return &meta, nil +} + +// Put writes meta information from RequestVars to the store. +func (s *MetaStore) Put(v *RequestVars) (*MetaObject, error) { + if !s.authenticate(v.Authorization) { + return nil, newAuthError() + } + + // Check if it exists first + if meta, err := s.Get(v); err == nil { + meta.Existing = true + return meta, nil + } + + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + meta := MetaObject{Oid: v.Oid, Size: v.Size} + err := enc.Encode(meta) + if err != nil { + return nil, err + } + + err = s.db.Update(func(tx *bolt.Tx) error { + bucket := tx.Bucket(objectsBucket) + if bucket == nil { + return errNoBucket + } + + err = bucket.Put([]byte(v.Oid), buf.Bytes()) + if err != nil { + return err + } + + return nil + }) + + if err != nil { + return nil, err + } + + return &meta, nil +} + +// Delete removes the meta information from RequestVars to the store. +func (s *MetaStore) Delete(v *RequestVars) error { + if !s.authenticate(v.Authorization) { + return newAuthError() + } + + err := s.db.Update(func(tx *bolt.Tx) error { + bucket := tx.Bucket(objectsBucket) + if bucket == nil { + return errNoBucket + } + + err := bucket.Delete([]byte(v.Oid)) + if err != nil { + return err + } + + return nil + }) + + return err +} + +// Close closes the underlying boltdb. +func (s *MetaStore) Close() { + s.db.Close() +} + +// AddUser adds user credentials to the meta store. +func (s *MetaStore) AddUser(user, pass string) error { + err := s.db.Update(func(tx *bolt.Tx) error { + bucket := tx.Bucket(usersBucket) + if bucket == nil { + return errNoBucket + } + + err := bucket.Put([]byte(user), []byte(pass)) + if err != nil { + return err + } + return nil + }) + + return err +} + +// DeleteUser removes user credentials from the meta store. +func (s *MetaStore) DeleteUser(user string) error { + err := s.db.Update(func(tx *bolt.Tx) error { + bucket := tx.Bucket(usersBucket) + if bucket == nil { + return errNoBucket + } + + err := bucket.Delete([]byte(user)) + return err + }) + + return err +} + +// MetaUser encapsulates information about a meta store user +type MetaUser struct { + Name string +} + +// Users returns all MetaUsers in the meta store +func (s *MetaStore) Users() ([]*MetaUser, error) { + var users []*MetaUser + + err := s.db.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket(usersBucket) + if bucket == nil { + return errNoBucket + } + + bucket.ForEach(func(k, v []byte) error { + users = append(users, &MetaUser{string(k)}) + return nil + }) + return nil + }) + + return users, err +} + +// Objects returns all MetaObjects in the meta store +func (s *MetaStore) Objects() ([]*MetaObject, error) { + var objects []*MetaObject + + err := s.db.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket(objectsBucket) + if bucket == nil { + return errNoBucket + } + + bucket.ForEach(func(k, v []byte) error { + var meta MetaObject + dec := gob.NewDecoder(bytes.NewBuffer(v)) + err := dec.Decode(&meta) + if err != nil { + return err + } + objects = append(objects, &meta) + return nil + }) + return nil + }) + + return objects, err +} + +// authenticate uses the authorization string to determine whether +// or not to proceed. This server assumes an HTTP Basic auth format. +func (s *MetaStore) authenticate(authorization string) bool { + if Config.IsPublic() { + return true + } + + if authorization == "" { + return false + } + + if !strings.HasPrefix(authorization, "Basic ") { + return false + } + + c, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(authorization, "Basic ")) + if err != nil { + return false + } + cs := string(c) + i := strings.IndexByte(cs, ':') + if i < 0 { + return false + } + user, password := cs[:i], cs[i+1:] + + // check Basic Authentication (admin) + ok := checkBasicAuth(user, password, true) + if ok { + return true + } + + value := "" + + s.db.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket(usersBucket) + if bucket == nil { + return errNoBucket + } + + value = string(bucket.Get([]byte(user))) + return nil + }) + + if value != "" && value == password { + return true + } + return false +} + +type authError struct { + error +} + +func (e authError) AuthError() bool { + return true +} + +func newAuthError() error { + return authError{errors.New("Forbidden")} +} diff --git a/modules/lfs/meta_store_test.go b/modules/lfs/meta_store_test.go new file mode 100644 index 0000000000000..19ae46d23fb7d --- /dev/null +++ b/modules/lfs/meta_store_test.go @@ -0,0 +1,111 @@ +package main + +import ( + "fmt" + "os" + "testing" +) + +var ( + metaStoreTest *MetaStore +) + +func TestGetWithAuth(t *testing.T) { + setupMeta() + defer teardownMeta() + + meta, err := metaStoreTest.Get(&RequestVars{Authorization: testAuth, Oid: contentOid}) + if err != nil { + t.Fatalf("Error retreiving meta: %s", err) + } + + if meta.Oid != contentOid { + t.Errorf("expected to get content oid, got: %s", meta.Oid) + } + + if meta.Size != contentSize { + t.Errorf("expected to get content size, got: %d", meta.Size) + } +} + +func TestGetWithoutAuth(t *testing.T) { + setupMeta() + defer teardownMeta() + + _, err := metaStoreTest.Get(&RequestVars{Authorization: badAuth, Oid: contentOid}) + if !isAuthError(err) { + t.Errorf("expected auth error, got: %s", err) + } +} + +func TestPutWithAuth(t *testing.T) { + setupMeta() + defer teardownMeta() + + meta, err := metaStoreTest.Put(&RequestVars{Authorization: testAuth, Oid: nonexistingOid, Size: 42}) + if err != nil { + t.Errorf("expected put to succeed, got : %s", err) + } + + if meta.Existing { + t.Errorf("expected meta to not have existed") + } + + meta, err = metaStoreTest.Get(&RequestVars{Authorization: testAuth, Oid: nonexistingOid}) + if err != nil { + t.Errorf("expected to be able to retreive new put, got : %s", err) + } + + if meta.Oid != nonexistingOid { + t.Errorf("expected oids to match, got: %s", meta.Oid) + } + + if meta.Size != 42 { + t.Errorf("expected sizes to match, got: %d", meta.Size) + } + + meta, err = metaStoreTest.Put(&RequestVars{Authorization: testAuth, Oid: nonexistingOid, Size: 42}) + if err != nil { + t.Errorf("expected put to succeed, got : %s", err) + } + + if !meta.Existing { + t.Errorf("expected meta to now exist") + } +} + +func TestPuthWithoutAuth(t *testing.T) { + setupMeta() + defer teardownMeta() + + _, err := metaStoreTest.Put(&RequestVars{Authorization: badAuth, Oid: contentOid, Size: 42}) + if !isAuthError(err) { + t.Errorf("expected auth error, got: %s", err) + } +} + +func setupMeta() { + store, err := NewMetaStore("test-meta-store.db") + if err != nil { + fmt.Printf("error initializing test meta store: %s\n", err) + os.Exit(1) + } + + metaStoreTest = store + if err := metaStoreTest.AddUser(testUser, testPass); err != nil { + teardownMeta() + fmt.Printf("error adding test user to meta store: %s\n", err) + os.Exit(1) + } + + rv := &RequestVars{Authorization: testAuth, Oid: contentOid, Size: contentSize} + if _, err := metaStoreTest.Put(rv); err != nil { + teardownMeta() + fmt.Printf("error seeding test meta store: %s\n", err) + os.Exit(1) + } +} + +func teardownMeta() { + os.RemoveAll("test-meta-store.db") +} diff --git a/modules/lfs/server.go b/modules/lfs/server.go new file mode 100644 index 0000000000000..1340784a5523d --- /dev/null +++ b/modules/lfs/server.go @@ -0,0 +1,400 @@ +package main + +import ( + "crypto/rand" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "regexp" + "strconv" + "strings" + "time" + + "github.com/gorilla/context" + "github.com/gorilla/mux" +) + +// RequestVars contain variables from the HTTP request. Variables from routing, json body decoding, and +// some headers are stored. +type RequestVars struct { + Oid string + Size int64 + User string + Password string + Repo string + Authorization string +} + +type BatchVars struct { + Transfers []string `json:"transfers,omitempty"` + Operation string `json:"operation"` + Objects []*RequestVars `json:"objects"` +} + +// MetaObject is object metadata as seen by the object and metadata stores. +type MetaObject struct { + Oid string `json:"oid"` + Size int64 `json:"size"` + Existing bool +} + +type BatchResponse struct { + Transfer string `json:"transfer,omitempty"` + Objects []*Representation `json:"objects"` +} + +// Representation is object medata as seen by clients of the lfs server. +type Representation struct { + Oid string `json:"oid"` + Size int64 `json:"size"` + Actions map[string]*link `json:"actions"` + Error *ObjectError `json:"error,omitempty"` +} + +type ObjectError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +// ObjectLink builds a URL linking to the object. +func (v *RequestVars) ObjectLink() string { + path := "" + + if len(v.User) > 0 { + path += fmt.Sprintf("/%s", v.User) + } + + if len(v.Repo) > 0 { + path += fmt.Sprintf("/%s", v.Repo) + } + + path += fmt.Sprintf("/objects/%s", v.Oid) + + if Config.IsHTTPS() { + return fmt.Sprintf("%s://%s%s", Config.Scheme, Config.Host, path) + } + + return fmt.Sprintf("http://%s%s", Config.Host, path) +} + +// link provides a structure used to build a hypermedia representation of an HTTP link. +type link struct { + Href string `json:"href"` + Header map[string]string `json:"header,omitempty"` + ExpiresAt time.Time `json:"expires_at,omitempty"` +} + +// App links a Router, ContentStore, and MetaStore to provide the LFS server. +type App struct { + router *mux.Router + contentStore *ContentStore + metaStore *MetaStore +} + +// NewApp creates a new App using the ContentStore and MetaStore provided +func NewApp(content *ContentStore, meta *MetaStore) *App { + app := &App{contentStore: content, metaStore: meta} + + r := mux.NewRouter() + + r.HandleFunc("/{user}/{repo}/objects/batch", app.BatchHandler).Methods("POST").MatcherFunc(MetaMatcher) + route := "/{user}/{repo}/objects/{oid}" + r.HandleFunc(route, app.GetContentHandler).Methods("GET", "HEAD").MatcherFunc(ContentMatcher) + r.HandleFunc(route, app.GetMetaHandler).Methods("GET", "HEAD").MatcherFunc(MetaMatcher) + r.HandleFunc(route, app.PutHandler).Methods("PUT").MatcherFunc(ContentMatcher) + + r.HandleFunc("/{user}/{repo}/objects", app.PostHandler).Methods("POST").MatcherFunc(MetaMatcher) + + r.HandleFunc("/objects/batch", app.BatchHandler).Methods("POST").MatcherFunc(MetaMatcher) + route = "/objects/{oid}" + r.HandleFunc(route, app.GetContentHandler).Methods("GET", "HEAD").MatcherFunc(ContentMatcher) + r.HandleFunc(route, app.GetMetaHandler).Methods("GET", "HEAD").MatcherFunc(MetaMatcher) + r.HandleFunc(route, app.PutHandler).Methods("PUT").MatcherFunc(ContentMatcher) + + r.HandleFunc("/objects", app.PostHandler).Methods("POST").MatcherFunc(MetaMatcher) + + app.addMgmt(r) + + app.router = r + + return app +} + +func (a *App) ServeHTTP(w http.ResponseWriter, r *http.Request) { + b := make([]byte, 16) + _, err := rand.Read(b) + if err == nil { + context.Set(r, "RequestID", fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])) + } + + a.router.ServeHTTP(w, r) +} + +// Serve calls http.Serve with the provided Listener and the app's router +func (a *App) Serve(l net.Listener) error { + return http.Serve(l, a) +} + +// GetContentHandler gets the content from the content store +func (a *App) GetContentHandler(w http.ResponseWriter, r *http.Request) { + rv := unpack(r) + meta, err := a.metaStore.Get(rv) + if err != nil { + if isAuthError(err) { + requireAuth(w, r) + } else { + writeStatus(w, r, 404) + } + return + } + + // Support resume download using Range header + var fromByte int64 + statusCode := 200 + if rangeHdr := r.Header.Get("Range"); rangeHdr != "" { + regex := regexp.MustCompile(`bytes=(\d+)\-.*`) + match := regex.FindStringSubmatch(rangeHdr) + if match != nil && len(match) > 1 { + statusCode = 206 + fromByte, _ = strconv.ParseInt(match[1], 10, 32) + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", fromByte, meta.Size-1, int64(meta.Size)-fromByte)) + } + } + + content, err := a.contentStore.Get(meta, fromByte) + if err != nil { + writeStatus(w, r, 404) + return + } + + w.WriteHeader(statusCode) + io.Copy(w, content) + logRequest(r, statusCode) +} + +// GetMetaHandler retrieves metadata about the object +func (a *App) GetMetaHandler(w http.ResponseWriter, r *http.Request) { + rv := unpack(r) + meta, err := a.metaStore.Get(rv) + if err != nil { + if isAuthError(err) { + requireAuth(w, r) + } else { + writeStatus(w, r, 404) + } + return + } + + w.Header().Set("Content-Type", metaMediaType) + + if r.Method == "GET" { + enc := json.NewEncoder(w) + enc.Encode(a.Represent(rv, meta, true, false)) + } + + logRequest(r, 200) +} + +// PostHandler instructs the client how to upload data +func (a *App) PostHandler(w http.ResponseWriter, r *http.Request) { + rv := unpack(r) + meta, err := a.metaStore.Put(rv) + if err != nil { + if isAuthError(err) { + requireAuth(w, r) + } else { + writeStatus(w, r, 404) + } + return + } + + w.Header().Set("Content-Type", metaMediaType) + + sentStatus := 202 + if meta.Existing && a.contentStore.Exists(meta) { + sentStatus = 200 + } + w.WriteHeader(sentStatus) + + enc := json.NewEncoder(w) + enc.Encode(a.Represent(rv, meta, meta.Existing, true)) + logRequest(r, sentStatus) +} + +// BatchHandler provides the batch api +func (a *App) BatchHandler(w http.ResponseWriter, r *http.Request) { + bv := unpackbatch(r) + + var responseObjects []*Representation + + // Create a response object + for _, object := range bv.Objects { + meta, err := a.metaStore.Get(object) + if err == nil && a.contentStore.Exists(meta) { // Object is found and exists + responseObjects = append(responseObjects, a.Represent(object, meta, true, false)) + continue + } + + if isAuthError(err) { + requireAuth(w, r) + return + } + + // Object is not found + meta, err = a.metaStore.Put(object) + if err == nil { + responseObjects = append(responseObjects, a.Represent(object, meta, meta.Existing, true)) + } + } + + w.Header().Set("Content-Type", metaMediaType) + + respobj := &BatchResponse{Objects: responseObjects} + + enc := json.NewEncoder(w) + enc.Encode(respobj) + logRequest(r, 200) +} + +// PutHandler receives data from the client and puts it into the content store +func (a *App) PutHandler(w http.ResponseWriter, r *http.Request) { + rv := unpack(r) + meta, err := a.metaStore.Get(rv) + if err != nil { + if isAuthError(err) { + requireAuth(w, r) + } else { + writeStatus(w, r, 404) + } + return + } + + if err := a.contentStore.Put(meta, r.Body); err != nil { + a.metaStore.Delete(rv) + w.WriteHeader(500) + fmt.Fprintf(w, `{"message":"%s"}`, err) + return + } + + logRequest(r, 200) +} + +// Represent takes a RequestVars and Meta and turns it into a Representation suitable +// for json encoding +func (a *App) Represent(rv *RequestVars, meta *MetaObject, download, upload bool) *Representation { + rep := &Representation{ + Oid: meta.Oid, + Size: meta.Size, + Actions: make(map[string]*link), + } + + header := make(map[string]string) + header["Accept"] = contentMediaType + if !Config.IsPublic() { + header["Authorization"] = rv.Authorization + } + if download { + rep.Actions["download"] = &link{Href: rv.ObjectLink(), Header: header} + } + + if upload { + rep.Actions["upload"] = &link{Href: rv.ObjectLink(), Header: header} + } + return rep +} + +// ContentMatcher provides a mux.MatcherFunc that only allows requests that contain +// an Accept header with the contentMediaType +func ContentMatcher(r *http.Request, m *mux.RouteMatch) bool { + mediaParts := strings.Split(r.Header.Get("Accept"), ";") + mt := mediaParts[0] + return mt == contentMediaType +} + +// MetaMatcher provides a mux.MatcherFunc that only allows requests that contain +// an Accept header with the metaMediaType +func MetaMatcher(r *http.Request, m *mux.RouteMatch) bool { + mediaParts := strings.Split(r.Header.Get("Accept"), ";") + mt := mediaParts[0] + return mt == metaMediaType +} + +func unpack(r *http.Request) *RequestVars { + vars := mux.Vars(r) + rv := &RequestVars{ + User: vars["user"], + Repo: vars["repo"], + Oid: vars["oid"], + Authorization: r.Header.Get("Authorization"), + } + + if r.Method == "POST" { // Maybe also check if +json + var p RequestVars + dec := json.NewDecoder(r.Body) + err := dec.Decode(&p) + if err != nil { + return rv + } + + rv.Oid = p.Oid + rv.Size = p.Size + } + + return rv +} + +// TODO cheap hack, unify with unpack +func unpackbatch(r *http.Request) *BatchVars { + vars := mux.Vars(r) + + var bv BatchVars + + dec := json.NewDecoder(r.Body) + err := dec.Decode(&bv) + if err != nil { + return &bv + } + + for i := 0; i < len(bv.Objects); i++ { + bv.Objects[i].User = vars["user"] + bv.Objects[i].Repo = vars["repo"] + bv.Objects[i].Authorization = r.Header.Get("Authorization") + } + + return &bv +} + +func writeStatus(w http.ResponseWriter, r *http.Request, status int) { + message := http.StatusText(status) + + mediaParts := strings.Split(r.Header.Get("Accept"), ";") + mt := mediaParts[0] + if strings.HasSuffix(mt, "+json") { + message = `{"message":"` + message + `"}` + } + + w.WriteHeader(status) + fmt.Fprint(w, message) + logRequest(r, status) +} + +func logRequest(r *http.Request, status int) { + logger.Log(kv{"method": r.Method, "url": r.URL, "status": status, "request_id": context.Get(r, "RequestID")}) +} + +func isAuthError(err error) bool { + type autherror interface { + AuthError() bool + } + if ae, ok := err.(autherror); ok { + return ae.AuthError() + } + return false +} + +func requireAuth(w http.ResponseWriter, r *http.Request) { + w.Header().Set("WWW-Authenticate", "Basic realm=git-lfs-server") + writeStatus(w, r, 401) +} diff --git a/modules/lfs/server_test.go b/modules/lfs/server_test.go new file mode 100644 index 0000000000000..4a235192d28ad --- /dev/null +++ b/modules/lfs/server_test.go @@ -0,0 +1,406 @@ +package main + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "os" + "testing" +) + +func TestGetAuthed(t *testing.T) { + req, err := http.NewRequest("GET", lfsServer.URL+"/user/repo/objects/"+contentOid, nil) + if err != nil { + t.Fatalf("request error: %s", err) + } + req.SetBasicAuth(testUser, testPass) + req.Header.Set("Accept", contentMediaType) + + res, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("response error: %s", err) + } + + if res.StatusCode != 200 { + t.Fatalf("expected status 302, got %d", res.StatusCode) + } + + by, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatalf("expected response to contain content, got error: %s", err) + } + + if string(by) != content { + t.Fatalf("expected content to be `content`, got: %s", string(by)) + } +} + +func TestGetAuthedWithRange(t *testing.T) { + req, err := http.NewRequest("GET", lfsServer.URL+"/user/repo/objects/"+contentOid, nil) + if err != nil { + t.Fatalf("request error: %s", err) + } + req.SetBasicAuth(testUser, testPass) + req.Header.Set("Accept", contentMediaType) + fromByte := 5 + req.Header.Set("Range", fmt.Sprintf("bytes=%d-", fromByte)) + + res, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("response error: %s", err) + } + + if res.StatusCode != 206 { + t.Fatalf("expected status 206, got %d", res.StatusCode) + } + if cr := res.Header.Get("Content-Range"); len(cr) > 0 { + expected := fmt.Sprintf("bytes %d-%d/%d", fromByte, len(content)-1, len(content)-fromByte) + if cr != expected { + t.Fatalf("expected Content-Range header of %q, got %q", expected, cr) + } + } else { + t.Fatalf("missing Content-Range header in response") + } + + by, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatalf("expected response to contain content, got error: %s", err) + } + + if string(by) != content[fromByte:] { + t.Fatalf("expected content to be `content`, got: %s", string(by)) + } +} + +func TestGetUnauthed(t *testing.T) { + req, err := http.NewRequest("GET", lfsServer.URL+"/user/repo/objects/"+contentOid, nil) + if err != nil { + t.Fatalf("request error: %s", err) + } + req.Header.Set("Accept", contentMediaType) + + res, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("response error: %s", err) + } + + if res.StatusCode != 401 { + t.Fatalf("expected status 401, got %d %s", res.StatusCode, req.URL) + } +} + +func TestGetMetaAuthed(t *testing.T) { + req, err := http.NewRequest("GET", lfsServer.URL+"/bilbo/repo/objects/"+contentOid, nil) + if err != nil { + t.Fatalf("request error: %s", err) + } + req.SetBasicAuth(testUser, testPass) + req.Header.Set("Accept", metaMediaType) + + res, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("response error: %s", err) + } + + if res.StatusCode != 200 { + t.Fatalf("expected status 200, got %d %s", res.StatusCode, req.URL) + } + + var meta Representation + dec := json.NewDecoder(res.Body) + dec.Decode(&meta) + + if meta.Oid != contentOid { + t.Fatalf("expected to see oid `%s` in meta, got: `%s`", contentOid, meta.Oid) + } + + if meta.Size != contentSize { + t.Fatalf("expected to see a size of `%d`, got: `%d`", contentSize, meta.Size) + } + + download := meta.Actions["download"] + if download.Href != "http://localhost:8080/bilbo/repo/objects/"+contentOid { + t.Fatalf("expected download link, got %s", download.Href) + } +} + +func TestGetMetaUnauthed(t *testing.T) { + req, err := http.NewRequest("GET", lfsServer.URL+"/user/repo/objects/"+contentOid, nil) + if err != nil { + t.Fatalf("request error: %s", err) + } + req.Header.Set("Accept", metaMediaType) + + res, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("response error: %s", err) + } + + if res.StatusCode != 401 { + t.Fatalf("expected status 401, got %d", res.StatusCode) + } +} + +func TestPostAuthedNewObject(t *testing.T) { + req, err := http.NewRequest("POST", lfsServer.URL+"/bilbo/repo/objects", nil) + if err != nil { + t.Fatalf("request error: %s", err) + } + req.SetBasicAuth(testUser, testPass) + req.Header.Set("Accept", metaMediaType) + + buf := bytes.NewBufferString(fmt.Sprintf(`{"oid":"%s", "size":1234}`, nonexistingOid)) + req.Body = ioutil.NopCloser(buf) + + res, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("response error: %s", err) + } + + if res.StatusCode != 202 { + t.Fatalf("expected status 202, got %d", res.StatusCode) + } + + var meta Representation + dec := json.NewDecoder(res.Body) + dec.Decode(&meta) + + if meta.Oid != nonexistingOid { + t.Fatalf("expected to see oid `%s` in meta, got: `%s`", nonexistingOid, meta.Oid) + } + + if meta.Size != 1234 { + t.Fatalf("expected to see a size of `1234`, got: `%d`", meta.Size) + } + + if download, ok := meta.Actions["download"]; ok { + t.Fatalf("expected POST to not contain a download link, got %s", download.Href) + } + + upload, ok := meta.Actions["upload"] + if !ok { + t.Fatal("expected upload link to be present") + } + + if upload.Href != "http://localhost:8080/bilbo/repo/objects/"+nonexistingOid { + t.Fatalf("expected upload link, got %s", upload.Href) + } +} + +func TestPostAuthedExistingObject(t *testing.T) { + req, err := http.NewRequest("POST", lfsServer.URL+"/bilbo/repo/objects", nil) + if err != nil { + t.Fatalf("request error: %s", err) + } + req.SetBasicAuth(testUser, testPass) + req.Header.Set("Accept", metaMediaType) + + buf := bytes.NewBufferString(fmt.Sprintf(`{"oid":"%s", "size":%d}`, contentOid, contentSize)) + req.Body = ioutil.NopCloser(buf) + + res, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("response error: %s", err) + } + + if res.StatusCode != 200 { + t.Fatalf("expected status 200, got %d", res.StatusCode) + } + + var meta Representation + dec := json.NewDecoder(res.Body) + dec.Decode(&meta) + + if meta.Oid != contentOid { + t.Fatalf("expected to see oid `%s` in meta, got: `%s`", contentOid, meta.Oid) + } + + if meta.Size != contentSize { + t.Fatalf("expected to see a size of `%d`, got: `%d`", contentSize, meta.Size) + } + + download := meta.Actions["download"] + if download.Href != "http://localhost:8080/bilbo/repo/objects/"+contentOid { + t.Fatalf("expected download link, got %s", download.Href) + } + + upload, ok := meta.Actions["upload"] + if !ok { + t.Fatalf("expected upload link to be present") + } + + if upload.Href != "http://localhost:8080/bilbo/repo/objects/"+contentOid { + t.Fatalf("expected upload link, got %s", upload.Href) + } +} + +func TestPostUnauthed(t *testing.T) { + req, err := http.NewRequest("POST", lfsServer.URL+"/bilbo/readonly/objects", nil) + if err != nil { + t.Fatalf("request error: %s", err) + } + req.Header.Set("Accept", metaMediaType) + + buf := bytes.NewBufferString(fmt.Sprintf(`{"oid":"%s", "size":%d}`, contentOid, contentSize)) + req.Body = ioutil.NopCloser(buf) + + res, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("response error: %s", err) + } + + if res.StatusCode != 401 { + t.Fatalf("expected status 401, got %d", res.StatusCode) + } +} + +func TestPut(t *testing.T) { + req, err := http.NewRequest("PUT", lfsServer.URL+"/user/repo/objects/"+contentOid, nil) + if err != nil { + t.Fatalf("request error: %s", err) + } + req.SetBasicAuth(testUser, testPass) + req.Header.Set("Accept", contentMediaType) + req.Header.Set("Content-Type", "application/octet-stream") + req.Body = ioutil.NopCloser(bytes.NewBuffer([]byte(content))) + + res, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("response error: %s", err) + } + + if res.StatusCode != 200 { + t.Fatalf("expected status 200, got %d", res.StatusCode) + } + + r, err := testContentStore.Get(&MetaObject{Oid: contentOid}, 0) + if err != nil { + t.Fatalf("error retreiving from content store: %s", err) + } + c, err := ioutil.ReadAll(r) + if err != nil { + t.Fatalf("error reading content: %s", err) + } + if string(c) != content { + t.Fatalf("expected content, got `%s`", string(c)) + } +} + +func TestMediaTypesRequired(t *testing.T) { + m := []string{"GET", "PUT", "POST", "HEAD"} + for _, method := range m { + req, err := http.NewRequest(method, lfsServer.URL+"/user/repo/objects/"+contentOid, nil) + if err != nil { + t.Fatalf("request error: %s", err) + } + req.SetBasicAuth(testUser, testPass) + res, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("response error: %s", err) + } + + if res.StatusCode != 404 { + t.Fatalf("expected status 404, got %d", res.StatusCode) + } + } +} + +func TestMediaTypesParsed(t *testing.T) { + req, err := http.NewRequest("GET", lfsServer.URL+"/user/repo/objects/"+contentOid, nil) + if err != nil { + t.Fatalf("request error: %s", err) + } + req.SetBasicAuth(testUser, testPass) + req.Header.Set("Accept", contentMediaType+"; charset=utf-8") + + res, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("response error: %s", err) + } + + if res.StatusCode != 200 { + t.Fatalf("expected status 200, got %d", res.StatusCode) + } +} + +var ( + lfsServer *httptest.Server + testMetaStore *MetaStore + testContentStore *ContentStore + testUser = "bilbo" + testPass = "baggins" + testAuth = fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(testUser+":"+testPass))) + badAuth = fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte("azog:defiler"))) + content = "this is my content" + contentSize = int64(len(content)) + contentOid = "f97e1b2936a56511b3b6efc99011758e4700d60fb1674d31445d1ee40b663f24" + nonexistingOid = "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f" +) + +func TestMain(m *testing.M) { + os.Remove("lfs-test.db") + + var err error + testMetaStore, err = NewMetaStore("lfs-test.db") + if err != nil { + fmt.Printf("Error creating meta store: %s", err) + os.Exit(1) + } + + testContentStore, err = NewContentStore("lfs-content-test") + if err != nil { + fmt.Printf("Error creating content store: %s", err) + os.Exit(1) + } + + if err := seedMetaStore(); err != nil { + fmt.Printf("Error seeding meta store: %s", err) + os.Exit(1) + } + + if err := seedContentStore(); err != nil { + fmt.Printf("Error seeding content store: %s", err) + os.Exit(1) + } + + app := NewApp(testContentStore, testMetaStore) + lfsServer = httptest.NewServer(app) + + logger = NewKVLogger(ioutil.Discard) + + ret := m.Run() + + lfsServer.Close() + testMetaStore.Close() + os.Remove("lfs-test.db") + os.RemoveAll("lfs-content-test") + + os.Exit(ret) +} + +func seedMetaStore() error { + if err := testMetaStore.AddUser(testUser, testPass); err != nil { + return err + } + + rv := &RequestVars{Authorization: testAuth, Oid: contentOid, Size: contentSize} + if _, err := testMetaStore.Put(rv); err != nil { + return err + } + + return nil +} + +func seedContentStore() error { + meta := &MetaObject{Oid: contentOid, Size: contentSize} + buf := bytes.NewBuffer([]byte(content)) + if err := testContentStore.Put(meta, buf); err != nil { + return err + } + + return nil +} diff --git a/modules/lfs/tracking_listener.go b/modules/lfs/tracking_listener.go new file mode 100644 index 0000000000000..bd76cd4081a2f --- /dev/null +++ b/modules/lfs/tracking_listener.go @@ -0,0 +1,95 @@ +package main + +import ( + "fmt" + "net" + "net/url" + "os" + "strconv" + "sync" +) + +// TrackingListener tracks incoming connections so that application shutdown can +// wait until all in progress connections are finished before exiting. +type TrackingListener struct { + wg sync.WaitGroup + connections map[net.Conn]bool + cm sync.Mutex + net.Listener +} + +// NewTrackingListener creates a new TrackingListener, listening on the supplied +// address. +func NewTrackingListener(addr string) (*TrackingListener, error) { + var listener net.Listener + + a, err := url.Parse(addr) + if err != nil { + return nil, err + } + + switch a.Scheme { + case "fd": + fd, err := strconv.Atoi(a.Host) + if err != nil { + return nil, err + } + + f := os.NewFile(uintptr(fd), "trackinglistener") + listener, err = net.FileListener(f) + if err != nil { + return nil, err + } + case "tcp", "tcp4", "tcp6": + laddr, err := net.ResolveTCPAddr(a.Scheme, a.Host) + if err != nil { + return nil, err + } + + listener, err = net.ListenTCP(a.Scheme, laddr) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("Unsupported listener protocol: %s", a.Scheme) + } + + return &TrackingListener{Listener: listener, connections: make(map[net.Conn]bool)}, nil +} + +// Accept wraps the underlying net.Listener's Accept(), keeping track of all connections +// accepted. +func (l *TrackingListener) Accept() (net.Conn, error) { + l.wg.Add(1) + conn, err := l.Listener.Accept() + if err != nil { + l.wg.Done() + return nil, err + } + + c := &trackedConn{ + Conn: conn, + listener: l, + } + + return c, nil +} + +// WaitForChildren is called during shutdown. It will return once all the existing +// connections have finished. +func (l *TrackingListener) WaitForChildren() { + l.wg.Wait() + logger.Log(kv{"fn": "shutdown"}) +} + +type trackedConn struct { + net.Conn + listener *TrackingListener + once sync.Once +} + +func (c *trackedConn) Close() error { + c.once.Do(c.listener.wg.Done) + + return c.Conn.Close() +} From f03468288e4a23594302cbfe6c73e57146e1f83d Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Tue, 8 Nov 2016 22:03:33 +0100 Subject: [PATCH 02/37] Import github.com/dgrijalva/jwt-go into vendor/ --- vendor/github.com/dgrijalva/jwt-go/LICENSE | 8 + .../dgrijalva/jwt-go/MIGRATION_GUIDE.md | 97 ++++++++++++ vendor/github.com/dgrijalva/jwt-go/README.md | 85 ++++++++++ .../dgrijalva/jwt-go/VERSION_HISTORY.md | 105 +++++++++++++ vendor/github.com/dgrijalva/jwt-go/claims.go | 134 ++++++++++++++++ vendor/github.com/dgrijalva/jwt-go/doc.go | 4 + vendor/github.com/dgrijalva/jwt-go/ecdsa.go | 147 ++++++++++++++++++ .../dgrijalva/jwt-go/ecdsa_utils.go | 67 ++++++++ vendor/github.com/dgrijalva/jwt-go/errors.go | 59 +++++++ vendor/github.com/dgrijalva/jwt-go/hmac.go | 94 +++++++++++ .../github.com/dgrijalva/jwt-go/map_claims.go | 94 +++++++++++ vendor/github.com/dgrijalva/jwt-go/none.go | 52 +++++++ vendor/github.com/dgrijalva/jwt-go/parser.go | 131 ++++++++++++++++ vendor/github.com/dgrijalva/jwt-go/rsa.go | 100 ++++++++++++ vendor/github.com/dgrijalva/jwt-go/rsa_pss.go | 126 +++++++++++++++ .../github.com/dgrijalva/jwt-go/rsa_utils.go | 69 ++++++++ .../dgrijalva/jwt-go/signing_method.go | 35 +++++ vendor/github.com/dgrijalva/jwt-go/token.go | 108 +++++++++++++ vendor/vendor.json | 18 ++- 19 files changed, 1527 insertions(+), 6 deletions(-) create mode 100644 vendor/github.com/dgrijalva/jwt-go/LICENSE create mode 100644 vendor/github.com/dgrijalva/jwt-go/MIGRATION_GUIDE.md create mode 100644 vendor/github.com/dgrijalva/jwt-go/README.md create mode 100644 vendor/github.com/dgrijalva/jwt-go/VERSION_HISTORY.md create mode 100644 vendor/github.com/dgrijalva/jwt-go/claims.go create mode 100644 vendor/github.com/dgrijalva/jwt-go/doc.go create mode 100644 vendor/github.com/dgrijalva/jwt-go/ecdsa.go create mode 100644 vendor/github.com/dgrijalva/jwt-go/ecdsa_utils.go create mode 100644 vendor/github.com/dgrijalva/jwt-go/errors.go create mode 100644 vendor/github.com/dgrijalva/jwt-go/hmac.go create mode 100644 vendor/github.com/dgrijalva/jwt-go/map_claims.go create mode 100644 vendor/github.com/dgrijalva/jwt-go/none.go create mode 100644 vendor/github.com/dgrijalva/jwt-go/parser.go create mode 100644 vendor/github.com/dgrijalva/jwt-go/rsa.go create mode 100644 vendor/github.com/dgrijalva/jwt-go/rsa_pss.go create mode 100644 vendor/github.com/dgrijalva/jwt-go/rsa_utils.go create mode 100644 vendor/github.com/dgrijalva/jwt-go/signing_method.go create mode 100644 vendor/github.com/dgrijalva/jwt-go/token.go diff --git a/vendor/github.com/dgrijalva/jwt-go/LICENSE b/vendor/github.com/dgrijalva/jwt-go/LICENSE new file mode 100644 index 0000000000000..df83a9c2f0192 --- /dev/null +++ b/vendor/github.com/dgrijalva/jwt-go/LICENSE @@ -0,0 +1,8 @@ +Copyright (c) 2012 Dave Grijalva + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/vendor/github.com/dgrijalva/jwt-go/MIGRATION_GUIDE.md b/vendor/github.com/dgrijalva/jwt-go/MIGRATION_GUIDE.md new file mode 100644 index 0000000000000..7fc1f793cbc4e --- /dev/null +++ b/vendor/github.com/dgrijalva/jwt-go/MIGRATION_GUIDE.md @@ -0,0 +1,97 @@ +## Migration Guide from v2 -> v3 + +Version 3 adds several new, frequently requested features. To do so, it introduces a few breaking changes. We've worked to keep these as minimal as possible. This guide explains the breaking changes and how you can quickly update your code. + +### `Token.Claims` is now an interface type + +The most requested feature from the 2.0 verison of this library was the ability to provide a custom type to the JSON parser for claims. This was implemented by introducing a new interface, `Claims`, to replace `map[string]interface{}`. We also included two concrete implementations of `Claims`: `MapClaims` and `StandardClaims`. + +`MapClaims` is an alias for `map[string]interface{}` with built in validation behavior. It is the default claims type when using `Parse`. The usage is unchanged except you must type cast the claims property. + +The old example for parsing a token looked like this.. + +```go + if token, err := jwt.Parse(tokenString, keyLookupFunc); err == nil { + fmt.Printf("Token for user %v expires %v", token.Claims["user"], token.Claims["exp"]) + } +``` + +is now directly mapped to... + +```go + if token, err := jwt.Parse(tokenString, keyLookupFunc); err == nil { + claims := token.Claims.(jwt.MapClaims) + fmt.Printf("Token for user %v expires %v", claims["user"], claims["exp"]) + } +``` + +`StandardClaims` is designed to be embedded in your custom type. You can supply a custom claims type with the new `ParseWithClaims` function. Here's an example of using a custom claims type. + +```go + type MyCustomClaims struct { + User string + *StandardClaims + } + + if token, err := jwt.ParseWithClaims(tokenString, &MyCustomClaims{}, keyLookupFunc); err == nil { + claims := token.Claims.(*MyCustomClaims) + fmt.Printf("Token for user %v expires %v", claims.User, claims.StandardClaims.ExpiresAt) + } +``` + +### `ParseFromRequest` has been moved + +To keep this library focused on the tokens without becoming overburdened with complex request processing logic, `ParseFromRequest` and its new companion `ParseFromRequestWithClaims` have been moved to a subpackage, `request`. The method signatues have also been augmented to receive a new argument: `Extractor`. + +`Extractors` do the work of picking the token string out of a request. The interface is simple and composable. + +This simple parsing example: + +```go + if token, err := jwt.ParseFromRequest(tokenString, req, keyLookupFunc); err == nil { + fmt.Printf("Token for user %v expires %v", token.Claims["user"], token.Claims["exp"]) + } +``` + +is directly mapped to: + +```go + if token, err := request.ParseFromRequest(req, request.OAuth2Extractor, keyLookupFunc); err == nil { + claims := token.Claims.(jwt.MapClaims) + fmt.Printf("Token for user %v expires %v", claims["user"], claims["exp"]) + } +``` + +There are several concrete `Extractor` types provided for your convenience: + +* `HeaderExtractor` will search a list of headers until one contains content. +* `ArgumentExtractor` will search a list of keys in request query and form arguments until one contains content. +* `MultiExtractor` will try a list of `Extractors` in order until one returns content. +* `AuthorizationHeaderExtractor` will look in the `Authorization` header for a `Bearer` token. +* `OAuth2Extractor` searches the places an OAuth2 token would be specified (per the spec): `Authorization` header and `access_token` argument +* `PostExtractionFilter` wraps an `Extractor`, allowing you to process the content before it's parsed. A simple example is stripping the `Bearer ` text from a header + + +### RSA signing methods no longer accept `[]byte` keys + +Due to a [critical vulnerability](https://auth0.com/blog/2015/03/31/critical-vulnerabilities-in-json-web-token-libraries/), we've decided the convenience of accepting `[]byte` instead of `rsa.PublicKey` or `rsa.PrivateKey` isn't worth the risk of misuse. + +To replace this behavior, we've added two helper methods: `ParseRSAPrivateKeyFromPEM(key []byte) (*rsa.PrivateKey, error)` and `ParseRSAPublicKeyFromPEM(key []byte) (*rsa.PublicKey, error)`. These are just simple helpers for unpacking PEM encoded PKCS1 and PKCS8 keys. If your keys are encoded any other way, all you need to do is convert them to the `crypto/rsa` package's types. + +```go + func keyLookupFunc(*Token) (interface{}, error) { + // Don't forget to validate the alg is what you expect: + if _, ok := token.Method.(*jwt.SigningMethodRSA); !ok { + return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"]) + } + + // Look up key + key, err := lookupPublicKey(token.Header["kid"]) + if err != nil { + return nil, err + } + + // Unpack key from PEM encoded PKCS8 + return jwt.ParseRSAPublicKeyFromPEM(key) + } +``` diff --git a/vendor/github.com/dgrijalva/jwt-go/README.md b/vendor/github.com/dgrijalva/jwt-go/README.md new file mode 100644 index 0000000000000..f48365fafbfda --- /dev/null +++ b/vendor/github.com/dgrijalva/jwt-go/README.md @@ -0,0 +1,85 @@ +A [go](http://www.golang.org) (or 'golang' for search engine friendliness) implementation of [JSON Web Tokens](http://self-issued.info/docs/draft-ietf-oauth-json-web-token.html) + +[![Build Status](https://travis-ci.org/dgrijalva/jwt-go.svg?branch=master)](https://travis-ci.org/dgrijalva/jwt-go) + +**BREAKING CHANGES:*** Version 3.0.0 is here. It includes _a lot_ of changes including a few that break the API. We've tried to break as few things as possible, so there should just be a few type signature changes. A full list of breaking changes is available in `VERSION_HISTORY.md`. See `MIGRATION_GUIDE.md` for more information on updating your code. + +**NOTICE:** A vulnerability in JWT was [recently published](https://auth0.com/blog/2015/03/31/critical-vulnerabilities-in-json-web-token-libraries/). As this library doesn't force users to validate the `alg` is what they expected, it's possible your usage is effected. There will be an update soon to remedy this, and it will likey require backwards-incompatible changes to the API. In the short term, please make sure your implementation verifies the `alg` is what you expect. + + +## What the heck is a JWT? + +JWT.io has [a great introduction](https://jwt.io/introduction) to JSON Web Tokens. + +In short, it's a signed JSON object that does something useful (for example, authentication). It's commonly used for `Bearer` tokens in Oauth 2. A token is made of three parts, separated by `.`'s. The first two parts are JSON objects, that have been [base64url](http://tools.ietf.org/html/rfc4648) encoded. The last part is the signature, encoded the same way. + +The first part is called the header. It contains the necessary information for verifying the last part, the signature. For example, which encryption method was used for signing and what key was used. + +The part in the middle is the interesting bit. It's called the Claims and contains the actual stuff you care about. Refer to [the RFC](http://self-issued.info/docs/draft-jones-json-web-token.html) for information about reserved keys and the proper way to add your own. + +## What's in the box? + +This library supports the parsing and verification as well as the generation and signing of JWTs. Current supported signing algorithms are HMAC SHA, RSA, RSA-PSS, and ECDSA, though hooks are present for adding your own. + +## Examples + +See [the project documentation](https://godoc.org/github.com/dgrijalva/jwt-go) for examples of usage: + +* [Simple example of parsing and validating a token](https://godoc.org/github.com/dgrijalva/jwt-go#example-Parse--Hmac) +* [Simple example of building and signing a token](https://godoc.org/github.com/dgrijalva/jwt-go#example-New--Hmac) +* [Directory of Examples](https://godoc.org/github.com/dgrijalva/jwt-go#pkg-examples) + +## Extensions + +This library publishes all the necessary components for adding your own signing methods. Simply implement the `SigningMethod` interface and register a factory method using `RegisterSigningMethod`. + +Here's an example of an extension that integrates with the Google App Engine signing tools: https://github.com/someone1/gcp-jwt-go + +## Compliance + +This library was last reviewed to comply with [RTF 7519](http://www.rfc-editor.org/info/rfc7519) dated May 2015 with a few notable differences: + +* In order to protect against accidental use of [Unsecured JWTs](http://self-issued.info/docs/draft-ietf-oauth-json-web-token.html#UnsecuredJWT), tokens using `alg=none` will only be accepted if the constant `jwt.UnsafeAllowNoneSignatureType` is provided as the key. + +## Project Status & Versioning + +This library is considered production ready. Feedback and feature requests are appreciated. The API should be considered stable. There should be very few backwards-incompatible changes outside of major version updates (and only with good reason). + +This project uses [Semantic Versioning 2.0.0](http://semver.org). Accepted pull requests will land on `master`. Periodically, versions will be tagged from `master`. You can find all the releases on [the project releases page](https://github.com/dgrijalva/jwt-go/releases). + +While we try to make it obvious when we make breaking changes, there isn't a great mechanism for pushing announcements out to users. You may want to use this alternative package include: `gopkg.in/dgrijalva/jwt-go.v2`. It will do the right thing WRT semantic versioning. + +## Usage Tips + +### Signing vs Encryption + +A token is simply a JSON object that is signed by its author. this tells you exactly two things about the data: + +* The author of the token was in the possession of the signing secret +* The data has not been modified since it was signed + +It's important to know that JWT does not provide encryption, which means anyone who has access to the token can read its contents. If you need to protect (encrypt) the data, there is a companion spec, `JWE`, that provides this functionality. JWE is currently outside the scope of this library. + +### Choosing a Signing Method + +There are several signing methods available, and you should probably take the time to learn about the various options before choosing one. The principal design decision is most likely going to be symmetric vs asymmetric. + +Symmetric signing methods, such as HSA, use only a single secret. This is probably the simplest signing method to use since any `[]byte` can be used as a valid secret. They are also slightly computationally faster to use, though this rarely is enough to matter. Symmetric signing methods work the best when both producers and consumers of tokens are trusted, or even the same system. Since the same secret is used to both sign and validate tokens, you can't easily distribute the key for validation. + +Asymmetric signing methods, such as RSA, use different keys for signing and verifying tokens. This makes it possible to produce tokens with a private key, and allow any consumer to access the public key for verification. + +### JWT and OAuth + +It's worth mentioning that OAuth and JWT are not the same thing. A JWT token is simply a signed JSON object. It can be used anywhere such a thing is useful. There is some confusion, though, as JWT is the most common type of bearer token used in OAuth2 authentication. + +Without going too far down the rabbit hole, here's a description of the interaction of these technologies: + +* OAuth is a protocol for allowing an identity provider to be separate from the service a user is logging in to. For example, whenever you use Facebook to log into a different service (Yelp, Spotify, etc), you are using OAuth. +* OAuth defines several options for passing around authentication data. One popular method is called a "bearer token". A bearer token is simply a string that _should_ only be held by an authenticated user. Thus, simply presenting this token proves your identity. You can probably derive from here why a JWT might make a good bearer token. +* Because bearer tokens are used for authentication, it's important they're kept secret. This is why transactions that use bearer tokens typically happen over SSL. + +## More + +Documentation can be found [on godoc.org](http://godoc.org/github.com/dgrijalva/jwt-go). + +The command line utility included in this project (cmd/jwt) provides a straightforward example of token creation and parsing as well as a useful tool for debugging your own integration. You'll also find several implementation examples in to documentation. diff --git a/vendor/github.com/dgrijalva/jwt-go/VERSION_HISTORY.md b/vendor/github.com/dgrijalva/jwt-go/VERSION_HISTORY.md new file mode 100644 index 0000000000000..b605b45093095 --- /dev/null +++ b/vendor/github.com/dgrijalva/jwt-go/VERSION_HISTORY.md @@ -0,0 +1,105 @@ +## `jwt-go` Version History + +#### 3.0.0 + +* **Compatibility Breaking Changes**: See MIGRATION_GUIDE.md for tips on updating your code + * Dropped support for `[]byte` keys when using RSA signing methods. This convenience feature could contribute to security vulnerabilities involving mismatched key types with signing methods. + * `ParseFromRequest` has been moved to `request` subpackage and usage has changed + * The `Claims` property on `Token` is now type `Claims` instead of `map[string]interface{}`. The default value is type `MapClaims`, which is an alias to `map[string]interface{}`. This makes it possible to use a custom type when decoding claims. +* Other Additions and Changes + * Added `Claims` interface type to allow users to decode the claims into a custom type + * Added `ParseWithClaims`, which takes a third argument of type `Claims`. Use this function instead of `Parse` if you have a custom type you'd like to decode into. + * Dramatically improved the functionality and flexibility of `ParseFromRequest`, which is now in the `request` subpackage + * Added `ParseFromRequestWithClaims` which is the `FromRequest` equivalent of `ParseWithClaims` + * Added new interface type `Extractor`, which is used for extracting JWT strings from http requests. Used with `ParseFromRequest` and `ParseFromRequestWithClaims`. + * Added several new, more specific, validation errors to error type bitmask + * Moved examples from README to executable example files + * Signing method registry is now thread safe + * Added new property to `ValidationError`, which contains the raw error returned by calls made by parse/verify (such as those returned by keyfunc or json parser) + +#### 2.7.0 + +This will likely be the last backwards compatible release before 3.0.0, excluding essential bug fixes. + +* Added new option `-show` to the `jwt` command that will just output the decoded token without verifying +* Error text for expired tokens includes how long it's been expired +* Fixed incorrect error returned from `ParseRSAPublicKeyFromPEM` +* Documentation updates + +#### 2.6.0 + +* Exposed inner error within ValidationError +* Fixed validation errors when using UseJSONNumber flag +* Added several unit tests + +#### 2.5.0 + +* Added support for signing method none. You shouldn't use this. The API tries to make this clear. +* Updated/fixed some documentation +* Added more helpful error message when trying to parse tokens that begin with `BEARER ` + +#### 2.4.0 + +* Added new type, Parser, to allow for configuration of various parsing parameters + * You can now specify a list of valid signing methods. Anything outside this set will be rejected. + * You can now opt to use the `json.Number` type instead of `float64` when parsing token JSON +* Added support for [Travis CI](https://travis-ci.org/dgrijalva/jwt-go) +* Fixed some bugs with ECDSA parsing + +#### 2.3.0 + +* Added support for ECDSA signing methods +* Added support for RSA PSS signing methods (requires go v1.4) + +#### 2.2.0 + +* Gracefully handle a `nil` `Keyfunc` being passed to `Parse`. Result will now be the parsed token and an error, instead of a panic. + +#### 2.1.0 + +Backwards compatible API change that was missed in 2.0.0. + +* The `SignedString` method on `Token` now takes `interface{}` instead of `[]byte` + +#### 2.0.0 + +There were two major reasons for breaking backwards compatibility with this update. The first was a refactor required to expand the width of the RSA and HMAC-SHA signing implementations. There will likely be no required code changes to support this change. + +The second update, while unfortunately requiring a small change in integration, is required to open up this library to other signing methods. Not all keys used for all signing methods have a single standard on-disk representation. Requiring `[]byte` as the type for all keys proved too limiting. Additionally, this implementation allows for pre-parsed tokens to be reused, which might matter in an application that parses a high volume of tokens with a small set of keys. Backwards compatibilty has been maintained for passing `[]byte` to the RSA signing methods, but they will also accept `*rsa.PublicKey` and `*rsa.PrivateKey`. + +It is likely the only integration change required here will be to change `func(t *jwt.Token) ([]byte, error)` to `func(t *jwt.Token) (interface{}, error)` when calling `Parse`. + +* **Compatibility Breaking Changes** + * `SigningMethodHS256` is now `*SigningMethodHMAC` instead of `type struct` + * `SigningMethodRS256` is now `*SigningMethodRSA` instead of `type struct` + * `KeyFunc` now returns `interface{}` instead of `[]byte` + * `SigningMethod.Sign` now takes `interface{}` instead of `[]byte` for the key + * `SigningMethod.Verify` now takes `interface{}` instead of `[]byte` for the key +* Renamed type `SigningMethodHS256` to `SigningMethodHMAC`. Specific sizes are now just instances of this type. + * Added public package global `SigningMethodHS256` + * Added public package global `SigningMethodHS384` + * Added public package global `SigningMethodHS512` +* Renamed type `SigningMethodRS256` to `SigningMethodRSA`. Specific sizes are now just instances of this type. + * Added public package global `SigningMethodRS256` + * Added public package global `SigningMethodRS384` + * Added public package global `SigningMethodRS512` +* Moved sample private key for HMAC tests from an inline value to a file on disk. Value is unchanged. +* Refactored the RSA implementation to be easier to read +* Exposed helper methods `ParseRSAPrivateKeyFromPEM` and `ParseRSAPublicKeyFromPEM` + +#### 1.0.2 + +* Fixed bug in parsing public keys from certificates +* Added more tests around the parsing of keys for RS256 +* Code refactoring in RS256 implementation. No functional changes + +#### 1.0.1 + +* Fixed panic if RS256 signing method was passed an invalid key + +#### 1.0.0 + +* First versioned release +* API stabilized +* Supports creating, signing, parsing, and validating JWT tokens +* Supports RS256 and HS256 signing methods \ No newline at end of file diff --git a/vendor/github.com/dgrijalva/jwt-go/claims.go b/vendor/github.com/dgrijalva/jwt-go/claims.go new file mode 100644 index 0000000000000..f0228f02e0337 --- /dev/null +++ b/vendor/github.com/dgrijalva/jwt-go/claims.go @@ -0,0 +1,134 @@ +package jwt + +import ( + "crypto/subtle" + "fmt" + "time" +) + +// For a type to be a Claims object, it must just have a Valid method that determines +// if the token is invalid for any supported reason +type Claims interface { + Valid() error +} + +// Structured version of Claims Section, as referenced at +// https://tools.ietf.org/html/rfc7519#section-4.1 +// See examples for how to use this with your own claim types +type StandardClaims struct { + Audience string `json:"aud,omitempty"` + ExpiresAt int64 `json:"exp,omitempty"` + Id string `json:"jti,omitempty"` + IssuedAt int64 `json:"iat,omitempty"` + Issuer string `json:"iss,omitempty"` + NotBefore int64 `json:"nbf,omitempty"` + Subject string `json:"sub,omitempty"` +} + +// Validates time based claims "exp, iat, nbf". +// There is no accounting for clock skew. +// As well, if any of the above claims are not in the token, it will still +// be considered a valid claim. +func (c StandardClaims) Valid() error { + vErr := new(ValidationError) + now := TimeFunc().Unix() + + // The claims below are optional, by default, so if they are set to the + // default value in Go, let's not fail the verification for them. + if c.VerifyExpiresAt(now, false) == false { + delta := time.Unix(now, 0).Sub(time.Unix(c.ExpiresAt, 0)) + vErr.Inner = fmt.Errorf("token is expired by %v", delta) + vErr.Errors |= ValidationErrorExpired + } + + if c.VerifyIssuedAt(now, false) == false { + vErr.Inner = fmt.Errorf("Token used before issued") + vErr.Errors |= ValidationErrorIssuedAt + } + + if c.VerifyNotBefore(now, false) == false { + vErr.Inner = fmt.Errorf("token is not valid yet") + vErr.Errors |= ValidationErrorNotValidYet + } + + if vErr.valid() { + return nil + } + + return vErr +} + +// Compares the aud claim against cmp. +// If required is false, this method will return true if the value matches or is unset +func (c *StandardClaims) VerifyAudience(cmp string, req bool) bool { + return verifyAud(c.Audience, cmp, req) +} + +// Compares the exp claim against cmp. +// If required is false, this method will return true if the value matches or is unset +func (c *StandardClaims) VerifyExpiresAt(cmp int64, req bool) bool { + return verifyExp(c.ExpiresAt, cmp, req) +} + +// Compares the iat claim against cmp. +// If required is false, this method will return true if the value matches or is unset +func (c *StandardClaims) VerifyIssuedAt(cmp int64, req bool) bool { + return verifyIat(c.IssuedAt, cmp, req) +} + +// Compares the iss claim against cmp. +// If required is false, this method will return true if the value matches or is unset +func (c *StandardClaims) VerifyIssuer(cmp string, req bool) bool { + return verifyIss(c.Issuer, cmp, req) +} + +// Compares the nbf claim against cmp. +// If required is false, this method will return true if the value matches or is unset +func (c *StandardClaims) VerifyNotBefore(cmp int64, req bool) bool { + return verifyNbf(c.NotBefore, cmp, req) +} + +// ----- helpers + +func verifyAud(aud string, cmp string, required bool) bool { + if aud == "" { + return !required + } + if subtle.ConstantTimeCompare([]byte(aud), []byte(cmp)) != 0 { + return true + } else { + return false + } +} + +func verifyExp(exp int64, now int64, required bool) bool { + if exp == 0 { + return !required + } + return now <= exp +} + +func verifyIat(iat int64, now int64, required bool) bool { + if iat == 0 { + return !required + } + return now >= iat +} + +func verifyIss(iss string, cmp string, required bool) bool { + if iss == "" { + return !required + } + if subtle.ConstantTimeCompare([]byte(iss), []byte(cmp)) != 0 { + return true + } else { + return false + } +} + +func verifyNbf(nbf int64, now int64, required bool) bool { + if nbf == 0 { + return !required + } + return now >= nbf +} diff --git a/vendor/github.com/dgrijalva/jwt-go/doc.go b/vendor/github.com/dgrijalva/jwt-go/doc.go new file mode 100644 index 0000000000000..a86dc1a3b348c --- /dev/null +++ b/vendor/github.com/dgrijalva/jwt-go/doc.go @@ -0,0 +1,4 @@ +// Package jwt is a Go implementation of JSON Web Tokens: http://self-issued.info/docs/draft-jones-json-web-token.html +// +// See README.md for more info. +package jwt diff --git a/vendor/github.com/dgrijalva/jwt-go/ecdsa.go b/vendor/github.com/dgrijalva/jwt-go/ecdsa.go new file mode 100644 index 0000000000000..2f59a22236383 --- /dev/null +++ b/vendor/github.com/dgrijalva/jwt-go/ecdsa.go @@ -0,0 +1,147 @@ +package jwt + +import ( + "crypto" + "crypto/ecdsa" + "crypto/rand" + "errors" + "math/big" +) + +var ( + // Sadly this is missing from crypto/ecdsa compared to crypto/rsa + ErrECDSAVerification = errors.New("crypto/ecdsa: verification error") +) + +// Implements the ECDSA family of signing methods signing methods +type SigningMethodECDSA struct { + Name string + Hash crypto.Hash + KeySize int + CurveBits int +} + +// Specific instances for EC256 and company +var ( + SigningMethodES256 *SigningMethodECDSA + SigningMethodES384 *SigningMethodECDSA + SigningMethodES512 *SigningMethodECDSA +) + +func init() { + // ES256 + SigningMethodES256 = &SigningMethodECDSA{"ES256", crypto.SHA256, 32, 256} + RegisterSigningMethod(SigningMethodES256.Alg(), func() SigningMethod { + return SigningMethodES256 + }) + + // ES384 + SigningMethodES384 = &SigningMethodECDSA{"ES384", crypto.SHA384, 48, 384} + RegisterSigningMethod(SigningMethodES384.Alg(), func() SigningMethod { + return SigningMethodES384 + }) + + // ES512 + SigningMethodES512 = &SigningMethodECDSA{"ES512", crypto.SHA512, 66, 521} + RegisterSigningMethod(SigningMethodES512.Alg(), func() SigningMethod { + return SigningMethodES512 + }) +} + +func (m *SigningMethodECDSA) Alg() string { + return m.Name +} + +// Implements the Verify method from SigningMethod +// For this verify method, key must be an ecdsa.PublicKey struct +func (m *SigningMethodECDSA) Verify(signingString, signature string, key interface{}) error { + var err error + + // Decode the signature + var sig []byte + if sig, err = DecodeSegment(signature); err != nil { + return err + } + + // Get the key + var ecdsaKey *ecdsa.PublicKey + switch k := key.(type) { + case *ecdsa.PublicKey: + ecdsaKey = k + default: + return ErrInvalidKeyType + } + + if len(sig) != 2*m.KeySize { + return ErrECDSAVerification + } + + r := big.NewInt(0).SetBytes(sig[:m.KeySize]) + s := big.NewInt(0).SetBytes(sig[m.KeySize:]) + + // Create hasher + if !m.Hash.Available() { + return ErrHashUnavailable + } + hasher := m.Hash.New() + hasher.Write([]byte(signingString)) + + // Verify the signature + if verifystatus := ecdsa.Verify(ecdsaKey, hasher.Sum(nil), r, s); verifystatus == true { + return nil + } else { + return ErrECDSAVerification + } +} + +// Implements the Sign method from SigningMethod +// For this signing method, key must be an ecdsa.PrivateKey struct +func (m *SigningMethodECDSA) Sign(signingString string, key interface{}) (string, error) { + // Get the key + var ecdsaKey *ecdsa.PrivateKey + switch k := key.(type) { + case *ecdsa.PrivateKey: + ecdsaKey = k + default: + return "", ErrInvalidKeyType + } + + // Create the hasher + if !m.Hash.Available() { + return "", ErrHashUnavailable + } + + hasher := m.Hash.New() + hasher.Write([]byte(signingString)) + + // Sign the string and return r, s + if r, s, err := ecdsa.Sign(rand.Reader, ecdsaKey, hasher.Sum(nil)); err == nil { + curveBits := ecdsaKey.Curve.Params().BitSize + + if m.CurveBits != curveBits { + return "", ErrInvalidKey + } + + keyBytes := curveBits / 8 + if curveBits%8 > 0 { + keyBytes += 1 + } + + // We serialize the outpus (r and s) into big-endian byte arrays and pad + // them with zeros on the left to make sure the sizes work out. Both arrays + // must be keyBytes long, and the output must be 2*keyBytes long. + rBytes := r.Bytes() + rBytesPadded := make([]byte, keyBytes) + copy(rBytesPadded[keyBytes-len(rBytes):], rBytes) + + sBytes := s.Bytes() + sBytesPadded := make([]byte, keyBytes) + copy(sBytesPadded[keyBytes-len(sBytes):], sBytes) + + out := append(rBytesPadded, sBytesPadded...) + + return EncodeSegment(out), nil + } else { + return "", err + } +} diff --git a/vendor/github.com/dgrijalva/jwt-go/ecdsa_utils.go b/vendor/github.com/dgrijalva/jwt-go/ecdsa_utils.go new file mode 100644 index 0000000000000..d19624b7264fb --- /dev/null +++ b/vendor/github.com/dgrijalva/jwt-go/ecdsa_utils.go @@ -0,0 +1,67 @@ +package jwt + +import ( + "crypto/ecdsa" + "crypto/x509" + "encoding/pem" + "errors" +) + +var ( + ErrNotECPublicKey = errors.New("Key is not a valid ECDSA public key") + ErrNotECPrivateKey = errors.New("Key is not a valid ECDSA private key") +) + +// Parse PEM encoded Elliptic Curve Private Key Structure +func ParseECPrivateKeyFromPEM(key []byte) (*ecdsa.PrivateKey, error) { + var err error + + // Parse PEM block + var block *pem.Block + if block, _ = pem.Decode(key); block == nil { + return nil, ErrKeyMustBePEMEncoded + } + + // Parse the key + var parsedKey interface{} + if parsedKey, err = x509.ParseECPrivateKey(block.Bytes); err != nil { + return nil, err + } + + var pkey *ecdsa.PrivateKey + var ok bool + if pkey, ok = parsedKey.(*ecdsa.PrivateKey); !ok { + return nil, ErrNotECPrivateKey + } + + return pkey, nil +} + +// Parse PEM encoded PKCS1 or PKCS8 public key +func ParseECPublicKeyFromPEM(key []byte) (*ecdsa.PublicKey, error) { + var err error + + // Parse PEM block + var block *pem.Block + if block, _ = pem.Decode(key); block == nil { + return nil, ErrKeyMustBePEMEncoded + } + + // Parse the key + var parsedKey interface{} + if parsedKey, err = x509.ParsePKIXPublicKey(block.Bytes); err != nil { + if cert, err := x509.ParseCertificate(block.Bytes); err == nil { + parsedKey = cert.PublicKey + } else { + return nil, err + } + } + + var pkey *ecdsa.PublicKey + var ok bool + if pkey, ok = parsedKey.(*ecdsa.PublicKey); !ok { + return nil, ErrNotECPublicKey + } + + return pkey, nil +} diff --git a/vendor/github.com/dgrijalva/jwt-go/errors.go b/vendor/github.com/dgrijalva/jwt-go/errors.go new file mode 100644 index 0000000000000..1c93024aad2ea --- /dev/null +++ b/vendor/github.com/dgrijalva/jwt-go/errors.go @@ -0,0 +1,59 @@ +package jwt + +import ( + "errors" +) + +// Error constants +var ( + ErrInvalidKey = errors.New("key is invalid") + ErrInvalidKeyType = errors.New("key is of invalid type") + ErrHashUnavailable = errors.New("the requested hash function is unavailable") +) + +// The errors that might occur when parsing and validating a token +const ( + ValidationErrorMalformed uint32 = 1 << iota // Token is malformed + ValidationErrorUnverifiable // Token could not be verified because of signing problems + ValidationErrorSignatureInvalid // Signature validation failed + + // Standard Claim validation errors + ValidationErrorAudience // AUD validation failed + ValidationErrorExpired // EXP validation failed + ValidationErrorIssuedAt // IAT validation failed + ValidationErrorIssuer // ISS validation failed + ValidationErrorNotValidYet // NBF validation failed + ValidationErrorId // JTI validation failed + ValidationErrorClaimsInvalid // Generic claims validation error +) + +// Helper for constructing a ValidationError with a string error message +func NewValidationError(errorText string, errorFlags uint32) *ValidationError { + return &ValidationError{ + text: errorText, + Errors: errorFlags, + } +} + +// The error from Parse if token is not valid +type ValidationError struct { + Inner error // stores the error returned by external dependencies, i.e.: KeyFunc + Errors uint32 // bitfield. see ValidationError... constants + text string // errors that do not have a valid error just have text +} + +// Validation error is an error type +func (e ValidationError) Error() string { + if e.Inner != nil { + return e.Inner.Error() + } else if e.text != "" { + return e.text + } else { + return "token is invalid" + } +} + +// No errors +func (e *ValidationError) valid() bool { + return e.Errors == 0 +} diff --git a/vendor/github.com/dgrijalva/jwt-go/hmac.go b/vendor/github.com/dgrijalva/jwt-go/hmac.go new file mode 100644 index 0000000000000..c229919254556 --- /dev/null +++ b/vendor/github.com/dgrijalva/jwt-go/hmac.go @@ -0,0 +1,94 @@ +package jwt + +import ( + "crypto" + "crypto/hmac" + "errors" +) + +// Implements the HMAC-SHA family of signing methods signing methods +type SigningMethodHMAC struct { + Name string + Hash crypto.Hash +} + +// Specific instances for HS256 and company +var ( + SigningMethodHS256 *SigningMethodHMAC + SigningMethodHS384 *SigningMethodHMAC + SigningMethodHS512 *SigningMethodHMAC + ErrSignatureInvalid = errors.New("signature is invalid") +) + +func init() { + // HS256 + SigningMethodHS256 = &SigningMethodHMAC{"HS256", crypto.SHA256} + RegisterSigningMethod(SigningMethodHS256.Alg(), func() SigningMethod { + return SigningMethodHS256 + }) + + // HS384 + SigningMethodHS384 = &SigningMethodHMAC{"HS384", crypto.SHA384} + RegisterSigningMethod(SigningMethodHS384.Alg(), func() SigningMethod { + return SigningMethodHS384 + }) + + // HS512 + SigningMethodHS512 = &SigningMethodHMAC{"HS512", crypto.SHA512} + RegisterSigningMethod(SigningMethodHS512.Alg(), func() SigningMethod { + return SigningMethodHS512 + }) +} + +func (m *SigningMethodHMAC) Alg() string { + return m.Name +} + +// Verify the signature of HSXXX tokens. Returns nil if the signature is valid. +func (m *SigningMethodHMAC) Verify(signingString, signature string, key interface{}) error { + // Verify the key is the right type + keyBytes, ok := key.([]byte) + if !ok { + return ErrInvalidKeyType + } + + // Decode signature, for comparison + sig, err := DecodeSegment(signature) + if err != nil { + return err + } + + // Can we use the specified hashing method? + if !m.Hash.Available() { + return ErrHashUnavailable + } + + // This signing method is symmetric, so we validate the signature + // by reproducing the signature from the signing string and key, then + // comparing that against the provided signature. + hasher := hmac.New(m.Hash.New, keyBytes) + hasher.Write([]byte(signingString)) + if !hmac.Equal(sig, hasher.Sum(nil)) { + return ErrSignatureInvalid + } + + // No validation errors. Signature is good. + return nil +} + +// Implements the Sign method from SigningMethod for this signing method. +// Key must be []byte +func (m *SigningMethodHMAC) Sign(signingString string, key interface{}) (string, error) { + if keyBytes, ok := key.([]byte); ok { + if !m.Hash.Available() { + return "", ErrHashUnavailable + } + + hasher := hmac.New(m.Hash.New, keyBytes) + hasher.Write([]byte(signingString)) + + return EncodeSegment(hasher.Sum(nil)), nil + } + + return "", ErrInvalidKey +} diff --git a/vendor/github.com/dgrijalva/jwt-go/map_claims.go b/vendor/github.com/dgrijalva/jwt-go/map_claims.go new file mode 100644 index 0000000000000..291213c460d45 --- /dev/null +++ b/vendor/github.com/dgrijalva/jwt-go/map_claims.go @@ -0,0 +1,94 @@ +package jwt + +import ( + "encoding/json" + "errors" + // "fmt" +) + +// Claims type that uses the map[string]interface{} for JSON decoding +// This is the default claims type if you don't supply one +type MapClaims map[string]interface{} + +// Compares the aud claim against cmp. +// If required is false, this method will return true if the value matches or is unset +func (m MapClaims) VerifyAudience(cmp string, req bool) bool { + aud, _ := m["aud"].(string) + return verifyAud(aud, cmp, req) +} + +// Compares the exp claim against cmp. +// If required is false, this method will return true if the value matches or is unset +func (m MapClaims) VerifyExpiresAt(cmp int64, req bool) bool { + switch exp := m["exp"].(type) { + case float64: + return verifyExp(int64(exp), cmp, req) + case json.Number: + v, _ := exp.Int64() + return verifyExp(v, cmp, req) + } + return req == false +} + +// Compares the iat claim against cmp. +// If required is false, this method will return true if the value matches or is unset +func (m MapClaims) VerifyIssuedAt(cmp int64, req bool) bool { + switch iat := m["iat"].(type) { + case float64: + return verifyIat(int64(iat), cmp, req) + case json.Number: + v, _ := iat.Int64() + return verifyIat(v, cmp, req) + } + return req == false +} + +// Compares the iss claim against cmp. +// If required is false, this method will return true if the value matches or is unset +func (m MapClaims) VerifyIssuer(cmp string, req bool) bool { + iss, _ := m["iss"].(string) + return verifyIss(iss, cmp, req) +} + +// Compares the nbf claim against cmp. +// If required is false, this method will return true if the value matches or is unset +func (m MapClaims) VerifyNotBefore(cmp int64, req bool) bool { + switch nbf := m["nbf"].(type) { + case float64: + return verifyNbf(int64(nbf), cmp, req) + case json.Number: + v, _ := nbf.Int64() + return verifyNbf(v, cmp, req) + } + return req == false +} + +// Validates time based claims "exp, iat, nbf". +// There is no accounting for clock skew. +// As well, if any of the above claims are not in the token, it will still +// be considered a valid claim. +func (m MapClaims) Valid() error { + vErr := new(ValidationError) + now := TimeFunc().Unix() + + if m.VerifyExpiresAt(now, false) == false { + vErr.Inner = errors.New("Token is expired") + vErr.Errors |= ValidationErrorExpired + } + + if m.VerifyIssuedAt(now, false) == false { + vErr.Inner = errors.New("Token used before issued") + vErr.Errors |= ValidationErrorIssuedAt + } + + if m.VerifyNotBefore(now, false) == false { + vErr.Inner = errors.New("Token is not valid yet") + vErr.Errors |= ValidationErrorNotValidYet + } + + if vErr.valid() { + return nil + } + + return vErr +} diff --git a/vendor/github.com/dgrijalva/jwt-go/none.go b/vendor/github.com/dgrijalva/jwt-go/none.go new file mode 100644 index 0000000000000..f04d189d067be --- /dev/null +++ b/vendor/github.com/dgrijalva/jwt-go/none.go @@ -0,0 +1,52 @@ +package jwt + +// Implements the none signing method. This is required by the spec +// but you probably should never use it. +var SigningMethodNone *signingMethodNone + +const UnsafeAllowNoneSignatureType unsafeNoneMagicConstant = "none signing method allowed" + +var NoneSignatureTypeDisallowedError error + +type signingMethodNone struct{} +type unsafeNoneMagicConstant string + +func init() { + SigningMethodNone = &signingMethodNone{} + NoneSignatureTypeDisallowedError = NewValidationError("'none' signature type is not allowed", ValidationErrorSignatureInvalid) + + RegisterSigningMethod(SigningMethodNone.Alg(), func() SigningMethod { + return SigningMethodNone + }) +} + +func (m *signingMethodNone) Alg() string { + return "none" +} + +// Only allow 'none' alg type if UnsafeAllowNoneSignatureType is specified as the key +func (m *signingMethodNone) Verify(signingString, signature string, key interface{}) (err error) { + // Key must be UnsafeAllowNoneSignatureType to prevent accidentally + // accepting 'none' signing method + if _, ok := key.(unsafeNoneMagicConstant); !ok { + return NoneSignatureTypeDisallowedError + } + // If signing method is none, signature must be an empty string + if signature != "" { + return NewValidationError( + "'none' signing method with non-empty signature", + ValidationErrorSignatureInvalid, + ) + } + + // Accept 'none' signing method. + return nil +} + +// Only allow 'none' signing if UnsafeAllowNoneSignatureType is specified as the key +func (m *signingMethodNone) Sign(signingString string, key interface{}) (string, error) { + if _, ok := key.(unsafeNoneMagicConstant); ok { + return "", nil + } + return "", NoneSignatureTypeDisallowedError +} diff --git a/vendor/github.com/dgrijalva/jwt-go/parser.go b/vendor/github.com/dgrijalva/jwt-go/parser.go new file mode 100644 index 0000000000000..7bf1c4ea08422 --- /dev/null +++ b/vendor/github.com/dgrijalva/jwt-go/parser.go @@ -0,0 +1,131 @@ +package jwt + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" +) + +type Parser struct { + ValidMethods []string // If populated, only these methods will be considered valid + UseJSONNumber bool // Use JSON Number format in JSON decoder + SkipClaimsValidation bool // Skip claims validation during token parsing +} + +// Parse, validate, and return a token. +// keyFunc will receive the parsed token and should return the key for validating. +// If everything is kosher, err will be nil +func (p *Parser) Parse(tokenString string, keyFunc Keyfunc) (*Token, error) { + return p.ParseWithClaims(tokenString, MapClaims{}, keyFunc) +} + +func (p *Parser) ParseWithClaims(tokenString string, claims Claims, keyFunc Keyfunc) (*Token, error) { + parts := strings.Split(tokenString, ".") + if len(parts) != 3 { + return nil, NewValidationError("token contains an invalid number of segments", ValidationErrorMalformed) + } + + var err error + token := &Token{Raw: tokenString} + + // parse Header + var headerBytes []byte + if headerBytes, err = DecodeSegment(parts[0]); err != nil { + if strings.HasPrefix(strings.ToLower(tokenString), "bearer ") { + return token, NewValidationError("tokenstring should not contain 'bearer '", ValidationErrorMalformed) + } + return token, &ValidationError{Inner: err, Errors: ValidationErrorMalformed} + } + if err = json.Unmarshal(headerBytes, &token.Header); err != nil { + return token, &ValidationError{Inner: err, Errors: ValidationErrorMalformed} + } + + // parse Claims + var claimBytes []byte + token.Claims = claims + + if claimBytes, err = DecodeSegment(parts[1]); err != nil { + return token, &ValidationError{Inner: err, Errors: ValidationErrorMalformed} + } + dec := json.NewDecoder(bytes.NewBuffer(claimBytes)) + if p.UseJSONNumber { + dec.UseNumber() + } + // JSON Decode. Special case for map type to avoid weird pointer behavior + if c, ok := token.Claims.(MapClaims); ok { + err = dec.Decode(&c) + } else { + err = dec.Decode(&claims) + } + // Handle decode error + if err != nil { + return token, &ValidationError{Inner: err, Errors: ValidationErrorMalformed} + } + + // Lookup signature method + if method, ok := token.Header["alg"].(string); ok { + if token.Method = GetSigningMethod(method); token.Method == nil { + return token, NewValidationError("signing method (alg) is unavailable.", ValidationErrorUnverifiable) + } + } else { + return token, NewValidationError("signing method (alg) is unspecified.", ValidationErrorUnverifiable) + } + + // Verify signing method is in the required set + if p.ValidMethods != nil { + var signingMethodValid = false + var alg = token.Method.Alg() + for _, m := range p.ValidMethods { + if m == alg { + signingMethodValid = true + break + } + } + if !signingMethodValid { + // signing method is not in the listed set + return token, NewValidationError(fmt.Sprintf("signing method %v is invalid", alg), ValidationErrorSignatureInvalid) + } + } + + // Lookup key + var key interface{} + if keyFunc == nil { + // keyFunc was not provided. short circuiting validation + return token, NewValidationError("no Keyfunc was provided.", ValidationErrorUnverifiable) + } + if key, err = keyFunc(token); err != nil { + // keyFunc returned an error + return token, &ValidationError{Inner: err, Errors: ValidationErrorUnverifiable} + } + + vErr := &ValidationError{} + + // Validate Claims + if !p.SkipClaimsValidation { + if err := token.Claims.Valid(); err != nil { + + // If the Claims Valid returned an error, check if it is a validation error, + // If it was another error type, create a ValidationError with a generic ClaimsInvalid flag set + if e, ok := err.(*ValidationError); !ok { + vErr = &ValidationError{Inner: err, Errors: ValidationErrorClaimsInvalid} + } else { + vErr = e + } + } + } + + // Perform validation + token.Signature = parts[2] + if err = token.Method.Verify(strings.Join(parts[0:2], "."), token.Signature, key); err != nil { + vErr.Inner = err + vErr.Errors |= ValidationErrorSignatureInvalid + } + + if vErr.valid() { + token.Valid = true + return token, nil + } + + return token, vErr +} diff --git a/vendor/github.com/dgrijalva/jwt-go/rsa.go b/vendor/github.com/dgrijalva/jwt-go/rsa.go new file mode 100644 index 0000000000000..0ae0b1984e5ae --- /dev/null +++ b/vendor/github.com/dgrijalva/jwt-go/rsa.go @@ -0,0 +1,100 @@ +package jwt + +import ( + "crypto" + "crypto/rand" + "crypto/rsa" +) + +// Implements the RSA family of signing methods signing methods +type SigningMethodRSA struct { + Name string + Hash crypto.Hash +} + +// Specific instances for RS256 and company +var ( + SigningMethodRS256 *SigningMethodRSA + SigningMethodRS384 *SigningMethodRSA + SigningMethodRS512 *SigningMethodRSA +) + +func init() { + // RS256 + SigningMethodRS256 = &SigningMethodRSA{"RS256", crypto.SHA256} + RegisterSigningMethod(SigningMethodRS256.Alg(), func() SigningMethod { + return SigningMethodRS256 + }) + + // RS384 + SigningMethodRS384 = &SigningMethodRSA{"RS384", crypto.SHA384} + RegisterSigningMethod(SigningMethodRS384.Alg(), func() SigningMethod { + return SigningMethodRS384 + }) + + // RS512 + SigningMethodRS512 = &SigningMethodRSA{"RS512", crypto.SHA512} + RegisterSigningMethod(SigningMethodRS512.Alg(), func() SigningMethod { + return SigningMethodRS512 + }) +} + +func (m *SigningMethodRSA) Alg() string { + return m.Name +} + +// Implements the Verify method from SigningMethod +// For this signing method, must be an rsa.PublicKey structure. +func (m *SigningMethodRSA) Verify(signingString, signature string, key interface{}) error { + var err error + + // Decode the signature + var sig []byte + if sig, err = DecodeSegment(signature); err != nil { + return err + } + + var rsaKey *rsa.PublicKey + var ok bool + + if rsaKey, ok = key.(*rsa.PublicKey); !ok { + return ErrInvalidKeyType + } + + // Create hasher + if !m.Hash.Available() { + return ErrHashUnavailable + } + hasher := m.Hash.New() + hasher.Write([]byte(signingString)) + + // Verify the signature + return rsa.VerifyPKCS1v15(rsaKey, m.Hash, hasher.Sum(nil), sig) +} + +// Implements the Sign method from SigningMethod +// For this signing method, must be an rsa.PrivateKey structure. +func (m *SigningMethodRSA) Sign(signingString string, key interface{}) (string, error) { + var rsaKey *rsa.PrivateKey + var ok bool + + // Validate type of key + if rsaKey, ok = key.(*rsa.PrivateKey); !ok { + return "", ErrInvalidKey + } + + // Create the hasher + if !m.Hash.Available() { + return "", ErrHashUnavailable + } + + hasher := m.Hash.New() + hasher.Write([]byte(signingString)) + + // Sign the string and return the encoded bytes + if sigBytes, err := rsa.SignPKCS1v15(rand.Reader, rsaKey, m.Hash, hasher.Sum(nil)); err == nil { + return EncodeSegment(sigBytes), nil + } else { + return "", err + } +} diff --git a/vendor/github.com/dgrijalva/jwt-go/rsa_pss.go b/vendor/github.com/dgrijalva/jwt-go/rsa_pss.go new file mode 100644 index 0000000000000..10ee9db8a4ed6 --- /dev/null +++ b/vendor/github.com/dgrijalva/jwt-go/rsa_pss.go @@ -0,0 +1,126 @@ +// +build go1.4 + +package jwt + +import ( + "crypto" + "crypto/rand" + "crypto/rsa" +) + +// Implements the RSAPSS family of signing methods signing methods +type SigningMethodRSAPSS struct { + *SigningMethodRSA + Options *rsa.PSSOptions +} + +// Specific instances for RS/PS and company +var ( + SigningMethodPS256 *SigningMethodRSAPSS + SigningMethodPS384 *SigningMethodRSAPSS + SigningMethodPS512 *SigningMethodRSAPSS +) + +func init() { + // PS256 + SigningMethodPS256 = &SigningMethodRSAPSS{ + &SigningMethodRSA{ + Name: "PS256", + Hash: crypto.SHA256, + }, + &rsa.PSSOptions{ + SaltLength: rsa.PSSSaltLengthAuto, + Hash: crypto.SHA256, + }, + } + RegisterSigningMethod(SigningMethodPS256.Alg(), func() SigningMethod { + return SigningMethodPS256 + }) + + // PS384 + SigningMethodPS384 = &SigningMethodRSAPSS{ + &SigningMethodRSA{ + Name: "PS384", + Hash: crypto.SHA384, + }, + &rsa.PSSOptions{ + SaltLength: rsa.PSSSaltLengthAuto, + Hash: crypto.SHA384, + }, + } + RegisterSigningMethod(SigningMethodPS384.Alg(), func() SigningMethod { + return SigningMethodPS384 + }) + + // PS512 + SigningMethodPS512 = &SigningMethodRSAPSS{ + &SigningMethodRSA{ + Name: "PS512", + Hash: crypto.SHA512, + }, + &rsa.PSSOptions{ + SaltLength: rsa.PSSSaltLengthAuto, + Hash: crypto.SHA512, + }, + } + RegisterSigningMethod(SigningMethodPS512.Alg(), func() SigningMethod { + return SigningMethodPS512 + }) +} + +// Implements the Verify method from SigningMethod +// For this verify method, key must be an rsa.PublicKey struct +func (m *SigningMethodRSAPSS) Verify(signingString, signature string, key interface{}) error { + var err error + + // Decode the signature + var sig []byte + if sig, err = DecodeSegment(signature); err != nil { + return err + } + + var rsaKey *rsa.PublicKey + switch k := key.(type) { + case *rsa.PublicKey: + rsaKey = k + default: + return ErrInvalidKey + } + + // Create hasher + if !m.Hash.Available() { + return ErrHashUnavailable + } + hasher := m.Hash.New() + hasher.Write([]byte(signingString)) + + return rsa.VerifyPSS(rsaKey, m.Hash, hasher.Sum(nil), sig, m.Options) +} + +// Implements the Sign method from SigningMethod +// For this signing method, key must be an rsa.PrivateKey struct +func (m *SigningMethodRSAPSS) Sign(signingString string, key interface{}) (string, error) { + var rsaKey *rsa.PrivateKey + + switch k := key.(type) { + case *rsa.PrivateKey: + rsaKey = k + default: + return "", ErrInvalidKeyType + } + + // Create the hasher + if !m.Hash.Available() { + return "", ErrHashUnavailable + } + + hasher := m.Hash.New() + hasher.Write([]byte(signingString)) + + // Sign the string and return the encoded bytes + if sigBytes, err := rsa.SignPSS(rand.Reader, rsaKey, m.Hash, hasher.Sum(nil), m.Options); err == nil { + return EncodeSegment(sigBytes), nil + } else { + return "", err + } +} diff --git a/vendor/github.com/dgrijalva/jwt-go/rsa_utils.go b/vendor/github.com/dgrijalva/jwt-go/rsa_utils.go new file mode 100644 index 0000000000000..213a90dbbf8d5 --- /dev/null +++ b/vendor/github.com/dgrijalva/jwt-go/rsa_utils.go @@ -0,0 +1,69 @@ +package jwt + +import ( + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "errors" +) + +var ( + ErrKeyMustBePEMEncoded = errors.New("Invalid Key: Key must be PEM encoded PKCS1 or PKCS8 private key") + ErrNotRSAPrivateKey = errors.New("Key is not a valid RSA private key") + ErrNotRSAPublicKey = errors.New("Key is not a valid RSA public key") +) + +// Parse PEM encoded PKCS1 or PKCS8 private key +func ParseRSAPrivateKeyFromPEM(key []byte) (*rsa.PrivateKey, error) { + var err error + + // Parse PEM block + var block *pem.Block + if block, _ = pem.Decode(key); block == nil { + return nil, ErrKeyMustBePEMEncoded + } + + var parsedKey interface{} + if parsedKey, err = x509.ParsePKCS1PrivateKey(block.Bytes); err != nil { + if parsedKey, err = x509.ParsePKCS8PrivateKey(block.Bytes); err != nil { + return nil, err + } + } + + var pkey *rsa.PrivateKey + var ok bool + if pkey, ok = parsedKey.(*rsa.PrivateKey); !ok { + return nil, ErrNotRSAPrivateKey + } + + return pkey, nil +} + +// Parse PEM encoded PKCS1 or PKCS8 public key +func ParseRSAPublicKeyFromPEM(key []byte) (*rsa.PublicKey, error) { + var err error + + // Parse PEM block + var block *pem.Block + if block, _ = pem.Decode(key); block == nil { + return nil, ErrKeyMustBePEMEncoded + } + + // Parse the key + var parsedKey interface{} + if parsedKey, err = x509.ParsePKIXPublicKey(block.Bytes); err != nil { + if cert, err := x509.ParseCertificate(block.Bytes); err == nil { + parsedKey = cert.PublicKey + } else { + return nil, err + } + } + + var pkey *rsa.PublicKey + var ok bool + if pkey, ok = parsedKey.(*rsa.PublicKey); !ok { + return nil, ErrNotRSAPublicKey + } + + return pkey, nil +} diff --git a/vendor/github.com/dgrijalva/jwt-go/signing_method.go b/vendor/github.com/dgrijalva/jwt-go/signing_method.go new file mode 100644 index 0000000000000..ed1f212b21e1e --- /dev/null +++ b/vendor/github.com/dgrijalva/jwt-go/signing_method.go @@ -0,0 +1,35 @@ +package jwt + +import ( + "sync" +) + +var signingMethods = map[string]func() SigningMethod{} +var signingMethodLock = new(sync.RWMutex) + +// Implement SigningMethod to add new methods for signing or verifying tokens. +type SigningMethod interface { + Verify(signingString, signature string, key interface{}) error // Returns nil if signature is valid + Sign(signingString string, key interface{}) (string, error) // Returns encoded signature or error + Alg() string // returns the alg identifier for this method (example: 'HS256') +} + +// Register the "alg" name and a factory function for signing method. +// This is typically done during init() in the method's implementation +func RegisterSigningMethod(alg string, f func() SigningMethod) { + signingMethodLock.Lock() + defer signingMethodLock.Unlock() + + signingMethods[alg] = f +} + +// Get a signing method from an "alg" string +func GetSigningMethod(alg string) (method SigningMethod) { + signingMethodLock.RLock() + defer signingMethodLock.RUnlock() + + if methodF, ok := signingMethods[alg]; ok { + method = methodF() + } + return +} diff --git a/vendor/github.com/dgrijalva/jwt-go/token.go b/vendor/github.com/dgrijalva/jwt-go/token.go new file mode 100644 index 0000000000000..d637e0867c65b --- /dev/null +++ b/vendor/github.com/dgrijalva/jwt-go/token.go @@ -0,0 +1,108 @@ +package jwt + +import ( + "encoding/base64" + "encoding/json" + "strings" + "time" +) + +// TimeFunc provides the current time when parsing token to validate "exp" claim (expiration time). +// You can override it to use another time value. This is useful for testing or if your +// server uses a different time zone than your tokens. +var TimeFunc = time.Now + +// Parse methods use this callback function to supply +// the key for verification. The function receives the parsed, +// but unverified Token. This allows you to use properties in the +// Header of the token (such as `kid`) to identify which key to use. +type Keyfunc func(*Token) (interface{}, error) + +// A JWT Token. Different fields will be used depending on whether you're +// creating or parsing/verifying a token. +type Token struct { + Raw string // The raw token. Populated when you Parse a token + Method SigningMethod // The signing method used or to be used + Header map[string]interface{} // The first segment of the token + Claims Claims // The second segment of the token + Signature string // The third segment of the token. Populated when you Parse a token + Valid bool // Is the token valid? Populated when you Parse/Verify a token +} + +// Create a new Token. Takes a signing method +func New(method SigningMethod) *Token { + return NewWithClaims(method, MapClaims{}) +} + +func NewWithClaims(method SigningMethod, claims Claims) *Token { + return &Token{ + Header: map[string]interface{}{ + "typ": "JWT", + "alg": method.Alg(), + }, + Claims: claims, + Method: method, + } +} + +// Get the complete, signed token +func (t *Token) SignedString(key interface{}) (string, error) { + var sig, sstr string + var err error + if sstr, err = t.SigningString(); err != nil { + return "", err + } + if sig, err = t.Method.Sign(sstr, key); err != nil { + return "", err + } + return strings.Join([]string{sstr, sig}, "."), nil +} + +// Generate the signing string. This is the +// most expensive part of the whole deal. Unless you +// need this for something special, just go straight for +// the SignedString. +func (t *Token) SigningString() (string, error) { + var err error + parts := make([]string, 2) + for i, _ := range parts { + var jsonValue []byte + if i == 0 { + if jsonValue, err = json.Marshal(t.Header); err != nil { + return "", err + } + } else { + if jsonValue, err = json.Marshal(t.Claims); err != nil { + return "", err + } + } + + parts[i] = EncodeSegment(jsonValue) + } + return strings.Join(parts, "."), nil +} + +// Parse, validate, and return a token. +// keyFunc will receive the parsed token and should return the key for validating. +// If everything is kosher, err will be nil +func Parse(tokenString string, keyFunc Keyfunc) (*Token, error) { + return new(Parser).Parse(tokenString, keyFunc) +} + +func ParseWithClaims(tokenString string, claims Claims, keyFunc Keyfunc) (*Token, error) { + return new(Parser).ParseWithClaims(tokenString, claims, keyFunc) +} + +// Encode JWT specific base64url encoding with padding stripped +func EncodeSegment(seg []byte) string { + return strings.TrimRight(base64.URLEncoding.EncodeToString(seg), "=") +} + +// Decode JWT specific base64url encoding with padding stripped +func DecodeSegment(seg string) ([]byte, error) { + if l := len(seg) % 4; l > 0 { + seg += strings.Repeat("=", 4-l) + } + + return base64.URLEncoding.DecodeString(seg) +} diff --git a/vendor/vendor.json b/vendor/vendor.json index 5545a121aad65..cea9e3b50eceb 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -80,6 +80,12 @@ "revision": "e32ca5036449b7ea12c62ed761ea1ad7fc88a4e2", "revisionTime": "2016-11-28T23:08:40Z" }, + { + "checksumSHA1": "2Fy1Y6Z3lRRX1891WF/+HT4XS2I=", + "path": "github.com/dgrijalva/jwt-go", + "revision": "9ed569b5d1ac936e6494082958d63a6aa4fff99a", + "revisionTime": "2016-11-01T19:39:35Z" + }, { "checksumSHA1": "5ftkjfUwI9A6xCQ1PwIAd5+qlo0=", "path": "github.com/elazarl/go-bindata-assetfs", @@ -879,17 +885,17 @@ "revisionTime": "2016-10-31T15:37:30Z" }, { - "checksumSHA1": "1MGpGDQqnUoRpv7VEcQrXOBydXE=", - "path": "golang.org/x/crypto/pbkdf2", - "revision": "8e06e8ddd9629eb88639aba897641bff8031f1d3", - "revisionTime": "2016-09-10T18:59:01Z" - }, - { "checksumSHA1": "MCeXr2RNeiG1XG6V+er1OR0qyeo=", "path": "golang.org/x/crypto/md4", "revision": "ede567c8e044a5913dad1d1af3696d9da953104c", "revisionTime": "2016-11-04T19:41:44Z" }, + { + "checksumSHA1": "1MGpGDQqnUoRpv7VEcQrXOBydXE=", + "path": "golang.org/x/crypto/pbkdf2", + "revision": "8e06e8ddd9629eb88639aba897641bff8031f1d3", + "revisionTime": "2016-09-10T18:59:01Z" + }, { "checksumSHA1": "LlElMHeTC34ng8eHzjvtUhAgrr8=", "path": "golang.org/x/crypto/ssh", From 3d8fd95ac30fee045b05f2708197df866015ae71 Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Sun, 6 Nov 2016 16:52:45 +0100 Subject: [PATCH 03/37] Remove config, add JWT support from github.com/mgit-at/lfs-test-server Imported commit f0cdcc5a01599c5a955dc1bbf683bb4acecdba83 --- modules/lfs/config.go | 67 --------------------------------------- modules/lfs/meta_store.go | 42 +++++++++++++++++++++--- 2 files changed, 38 insertions(+), 71 deletions(-) delete mode 100644 modules/lfs/config.go diff --git a/modules/lfs/config.go b/modules/lfs/config.go deleted file mode 100644 index ee4bdff4d83a0..0000000000000 --- a/modules/lfs/config.go +++ /dev/null @@ -1,67 +0,0 @@ -package main - -import ( - "fmt" - "os" - "reflect" - "strings" -) - -// Configuration holds application configuration. Values will be pulled from -// environment variables, prefixed by keyPrefix. Default values can be added -// via tags. -type Configuration struct { - Listen string `config:"tcp://:8080"` - Host string `config:"localhost:8080"` - MetaDB string `config:"lfs.db"` - ContentPath string `config:"lfs-content"` - AdminUser string `config:""` - AdminPass string `config:""` - Cert string `config:""` - Key string `config:""` - Scheme string `config:"http"` - Public string `config:"public"` -} - -func (c *Configuration) IsHTTPS() bool { - return strings.Contains(Config.Scheme, "https") -} - -func (c *Configuration) IsPublic() bool { - switch Config.Public { - case "1", "true", "TRUE": - return true - } - return false -} - -// Config is the global app configuration -var Config = &Configuration{} - -const keyPrefix = "LFS" - -func init() { - te := reflect.TypeOf(Config).Elem() - ve := reflect.ValueOf(Config).Elem() - - for i := 0; i < te.NumField(); i++ { - sf := te.Field(i) - name := sf.Name - field := ve.FieldByName(name) - - envVar := strings.ToUpper(fmt.Sprintf("%s_%s", keyPrefix, name)) - env := os.Getenv(envVar) - tag := sf.Tag.Get("config") - - if env == "" && tag != "" { - env = tag - } - - field.SetString(env) - } - - if port := os.Getenv("PORT"); port != "" { - // If $PORT is set, override LFS_LISTEN. This is useful for deploying to Heroku. - Config.Listen = "tcp://:" + port - } -} diff --git a/modules/lfs/meta_store.go b/modules/lfs/meta_store.go index e189ded2e48a6..761f873c005cf 100644 --- a/modules/lfs/meta_store.go +++ b/modules/lfs/meta_store.go @@ -5,10 +5,12 @@ import ( "encoding/base64" "encoding/gob" "errors" + "fmt" "strings" "time" "github.com/boltdb/bolt" + "github.com/dgrijalva/jwt-go" ) // MetaStore implements a metadata storage. It stores user credentials and Meta information @@ -52,7 +54,7 @@ func NewMetaStore(dbFile string) (*MetaStore, error) { // Get retrieves the Meta information for an object given information in // RequestVars func (s *MetaStore) Get(v *RequestVars) (*MetaObject, error) { - if !s.authenticate(v.Authorization) { + if !s.authenticate(v.Authorization, false) { return nil, newAuthError() } meta, error := s.UnsafeGet(v) @@ -89,7 +91,7 @@ func (s *MetaStore) UnsafeGet(v *RequestVars) (*MetaObject, error) { // Put writes meta information from RequestVars to the store. func (s *MetaStore) Put(v *RequestVars) (*MetaObject, error) { - if !s.authenticate(v.Authorization) { + if !s.authenticate(v.Authorization, true) { return nil, newAuthError() } @@ -130,7 +132,7 @@ func (s *MetaStore) Put(v *RequestVars) (*MetaObject, error) { // Delete removes the meta information from RequestVars to the store. func (s *MetaStore) Delete(v *RequestVars) error { - if !s.authenticate(v.Authorization) { + if !s.authenticate(v.Authorization, true) { return newAuthError() } @@ -242,7 +244,7 @@ func (s *MetaStore) Objects() ([]*MetaObject, error) { // authenticate uses the authorization string to determine whether // or not to proceed. This server assumes an HTTP Basic auth format. -func (s *MetaStore) authenticate(authorization string) bool { +func (s *MetaStore) authenticate(authorization string, requireWrite bool) bool { if Config.IsPublic() { return true } @@ -251,6 +253,10 @@ func (s *MetaStore) authenticate(authorization string) bool { return false } + if s.authenticateToken(authorization, requireWrite) { + return true + } + if !strings.HasPrefix(authorization, "Basic ") { return false } @@ -290,6 +296,34 @@ func (s *MetaStore) authenticate(authorization string) bool { return false } +func (s *MetaStore) authenticateToken(authorization string, requireWrite bool) bool { + if !strings.HasPrefix(authorization, "Bearer ") { + return false + } + + token, err := jwt.Parse(authorization[7:], func(t *jwt.Token) (interface{}, error) { + if _, ok := t.Method.(*jwt.SigningMethodHMAC); !ok { + return nil, fmt.Errorf("unexpected signing method: %v", t.Header["alg"]) + } + return Config.DecodeSecret() + }) + if err != nil { + return false + } + claims, claimsOk := token.Claims.(jwt.MapClaims) + if !token.Valid || !claimsOk { + return false + } + + opstr, ok := claims["op"].(string) + if !ok { + return false + } + op := strings.ToLower(strings.TrimSpace(opstr)) + status := op == "upload" || (op == "download" && !requireWrite) + return status +} + type authError struct { error } From 7b3bfe80adc6b5982ea939accc6c70d658b7230a Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Sun, 6 Nov 2016 17:55:23 +0100 Subject: [PATCH 04/37] Add LFS settings --- modules/setting/setting.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index fad884ae1ee76..176f59f8ab1cf 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -5,7 +5,10 @@ package setting import ( + "crypto/rand" + "encoding/base64" "fmt" + "io" "net/mail" "net/url" "os" @@ -89,6 +92,13 @@ var ( MinimumKeySizes map[string]int `ini:"-"` } + LFS struct { + StartServer bool `ini:"LFS_START_SERVER"` + ContentPath string `ini:"LFS_CONTENT_PATH"` + JWTSecretBase64 string `ini:"LFS_JWT_SECRET"` + JWTSecretBytes []byte `ini:"-"` + } + // Security settings InstallLock bool SecretKey string @@ -583,6 +593,27 @@ please consider changing to GITEA_CUSTOM`) } } + if err = Cfg.Section("server").MapTo(&LFS); err != nil { + log.Fatal(4, "Fail to map LFS settings: %v", err) + } + + if LFS.StartServer { + if err := os.MkdirAll(LFS.ContentPath, 0700); err != nil { + log.Fatal(4, "Fail to create '%s': %v", LFS.ContentPath, err) + } + + LFS.JWTSecretBytes = make([]byte, 32) + n, err := base64.StdEncoding.Decode(LFS.JWTSecretBytes, []byte(LFS.JWTSecretBase64)) + + if err != nil || n != 32 { + log.Warn("Failed loading LFS JWT secret, generating runtime secret key") + _, err = io.ReadFull(rand.Reader, LFS.JWTSecretBytes) + if err != nil { + log.Fatal(4, "Error generating temporary JWT secret for LFS server: %v", err) + } + } + } + sec = Cfg.Section("security") InstallLock = sec.Key("INSTALL_LOCK").MustBool(false) SecretKey = sec.Key("SECRET_KEY").MustString("!#@FDEWREWR&*(") From 3727c212ff02cb24c8ba866ca130953d4eba0015 Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Sun, 6 Nov 2016 18:28:18 +0100 Subject: [PATCH 05/37] Add LFS meta object model --- models/lfs.go | 92 ++++++++++++++++++++++++++++++++++++++++++++++++ models/models.go | 2 +- 2 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 models/lfs.go diff --git a/models/lfs.go b/models/lfs.go new file mode 100644 index 0000000000000..592daabd584ed --- /dev/null +++ b/models/lfs.go @@ -0,0 +1,92 @@ +package models + +import ( + "errors" + "github.com/go-xorm/xorm" + "time" +) + +type LFSMetaObject struct { + ID int64 `xorm:"pk autoincr"` + Oid string `xorm:"UNIQUE NOT NULL"` + Size int64 `xorm:"NOT NULL"` + Existing bool `xorm:"-"` + Created time.Time `xorm:"-"` + CreatedUnix int64 +} + +var ( + ErrLFSObjectNotExist = errors.New("LFS Meta object does not exist") +) + +func NewLFSMetaObject(m *LFSMetaObject) (*LFSMetaObject, error) { + var err error + + has, err := x.Get(m) + if err != nil { + return nil, err + } + + if has { + m.Existing = true + return m, nil + } + + sess := x.NewSession() + defer sessionRelease(sess) + if err = sess.Begin(); err != nil { + return nil, err + } + + if _, err = sess.Insert(m); err != nil { + return nil, err + } + + return m, sess.Commit() +} + +func GetLFSMetaObjectByOid(oid string) (*LFSMetaObject, error) { + if len(oid) == 0 { + return nil, ErrLFSObjectNotExist + } + + m := &LFSMetaObject{Oid: oid} + has, err := x.Get(m) + if err != nil { + return nil, err + } else if !has { + return nil, ErrLFSObjectNotExist + } + return m, nil +} + +func RemoveLFSMetaObjectByOid(oid string) error { + if len(oid) == 0 { + return ErrLFSObjectNotExist + } + + sess := x.NewSession() + defer sessionRelease(sess) + if err := sess.Begin(); err != nil { + return err + } + + m := &LFSMetaObject{Oid: oid} + + if _, err := sess.Delete(m); err != nil { + return err + } + + return sess.Commit() +} + +func (m *LFSMetaObject) BeforeInsert() { + m.CreatedUnix = time.Now().Unix() +} + +func (m *LFSMetaObject) AfterSet(colName string, _ xorm.Cell) { + switch colName { + case "created_unix": + m.Created = time.Unix(m.CreatedUnix, 0).Local() + } +} diff --git a/models/models.go b/models/models.go index c9858952b57cc..20812d0ebff63 100644 --- a/models/models.go +++ b/models/models.go @@ -79,7 +79,7 @@ func init() { new(Mirror), new(Release), new(LoginSource), new(Webhook), new(UpdateTask), new(HookTask), new(Team), new(OrgUser), new(TeamUser), new(TeamRepo), - new(Notice), new(EmailAddress)) + new(Notice), new(EmailAddress), new(LFSMetaObject)) gonicNames := []string{"SSL", "UID"} for _, name := range gonicNames { From 2e433b92ee852937f6e99d5cb4490ab7fb37b4a9 Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Sun, 6 Nov 2016 22:42:12 +0100 Subject: [PATCH 06/37] Add LFS routes and initialization --- cmd/web.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/cmd/web.go b/cmd/web.go index 9776fdc11a1dc..4d0b002c835ba 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -17,6 +17,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/public" @@ -29,6 +30,7 @@ import ( "code.gitea.io/gitea/routers/org" "code.gitea.io/gitea/routers/repo" "code.gitea.io/gitea/routers/user" + "github.com/go-macaron/binding" "github.com/go-macaron/cache" "github.com/go-macaron/captcha" @@ -569,6 +571,29 @@ func runWeb(ctx *cli.Context) error { }) // ***** END: Repository ***** + // ***** START: LFS ***** + + if setting.LFS.StartServer { + + lfsHandler := lfs.NewLFSHandler() + + m.Group("/lfs", func() { + + m.Post("/:user/:repo/objects/batch", lfsHandler.BatchHandler) // TODO MetaMatcher + m.Any("/:user/:repo/objects/:oid", lfsHandler.ObjectOidHandler) + + m.Post("/:user/:repo/objects", lfsHandler.PostHandler) // TODO MetaMatcher + m.Post("/objects/batch", lfsHandler.BatchHandler) // TODO MetaMatcher + + m.Any("/objects/:oid", lfsHandler.ObjectOidHandler) + m.Post("/objects", lfsHandler.PostHandler) // TODO MetaMatcher + + }, ignSignInAndCsrf) + + } + + // ***** END: LFS ***** + m.Group("/api", func() { apiv1.RegisterRoutes(m) }, ignSignIn) From 9e8f91dcf433fd58a55ec431ea5bf05f451b1297 Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Sun, 6 Nov 2016 22:42:42 +0100 Subject: [PATCH 07/37] Adapt LFS module: handlers, routing, meta store --- modules/lfs/content_store.go | 9 +- modules/lfs/content_store_test.go | 17 +- modules/lfs/main.go | 112 --------- modules/lfs/meta_store.go | 337 ------------------------- modules/lfs/meta_store_test.go | 111 -------- modules/lfs/server.go | 353 +++++++++++++++----------- modules/lfs/server_test.go | 406 ------------------------------ modules/lfs/tracking_listener.go | 95 ------- 8 files changed, 216 insertions(+), 1224 deletions(-) delete mode 100644 modules/lfs/main.go delete mode 100644 modules/lfs/meta_store.go delete mode 100644 modules/lfs/meta_store_test.go delete mode 100644 modules/lfs/server_test.go delete mode 100644 modules/lfs/tracking_listener.go diff --git a/modules/lfs/content_store.go b/modules/lfs/content_store.go index aff6e87b5476f..3f31a0ede955e 100644 --- a/modules/lfs/content_store.go +++ b/modules/lfs/content_store.go @@ -1,9 +1,10 @@ -package main +package lfs import ( "crypto/sha256" "encoding/hex" "errors" + "github.com/go-gitea/gitea/models" "io" "os" "path/filepath" @@ -30,7 +31,7 @@ func NewContentStore(base string) (*ContentStore, error) { // Get takes a Meta object and retreives the content from the store, returning // it as an io.Reader. If fromByte > 0, the reader starts from that byte -func (s *ContentStore) Get(meta *MetaObject, fromByte int64) (io.Reader, error) { +func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.Reader, error) { path := filepath.Join(s.basePath, transformKey(meta.Oid)) f, err := os.Open(path) @@ -44,7 +45,7 @@ func (s *ContentStore) Get(meta *MetaObject, fromByte int64) (io.Reader, error) } // Put takes a Meta object and an io.Reader and writes the content to the store. -func (s *ContentStore) Put(meta *MetaObject, r io.Reader) error { +func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error { path := filepath.Join(s.basePath, transformKey(meta.Oid)) tmpPath := path + ".tmp" @@ -85,7 +86,7 @@ func (s *ContentStore) Put(meta *MetaObject, r io.Reader) error { } // Exists returns true if the object exists in the content store. -func (s *ContentStore) Exists(meta *MetaObject) bool { +func (s *ContentStore) Exists(meta *models.LFSMetaObject) bool { path := filepath.Join(s.basePath, transformKey(meta.Oid)) if _, err := os.Stat(path); os.IsNotExist(err) { return false diff --git a/modules/lfs/content_store_test.go b/modules/lfs/content_store_test.go index 9c196489e30c8..e4bebfa55aa21 100644 --- a/modules/lfs/content_store_test.go +++ b/modules/lfs/content_store_test.go @@ -1,8 +1,9 @@ -package main +package lfs import ( "bytes" "fmt" + "github.com/gogits/gogs/models" "io/ioutil" "os" "testing" @@ -14,7 +15,7 @@ func TestContenStorePut(t *testing.T) { setup() defer teardown() - m := &MetaObject{ + m := &models.LFSMetaObject{ Oid: "6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72", Size: 12, } @@ -35,7 +36,7 @@ func TestContenStorePutHashMismatch(t *testing.T) { setup() defer teardown() - m := &MetaObject{ + m := &models.LFSMetaObject{ Oid: "6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72", Size: 12, } @@ -56,7 +57,7 @@ func TestContenStorePutSizeMismatch(t *testing.T) { setup() defer teardown() - m := &MetaObject{ + m := &models.LFSMetaObject{ Oid: "6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72", Size: 14, } @@ -77,7 +78,7 @@ func TestContenStoreGet(t *testing.T) { setup() defer teardown() - m := &MetaObject{ + m := &models.LFSMetaObject{ Oid: "6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72", Size: 12, } @@ -103,7 +104,7 @@ func TestContenStoreGetWithRange(t *testing.T) { setup() defer teardown() - m := &MetaObject{ + m := &models.LFSMetaObject{ Oid: "6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72", Size: 12, } @@ -129,7 +130,7 @@ func TestContenStoreGetNonExisting(t *testing.T) { setup() defer teardown() - _, err := contentStore.Get(&MetaObject{Oid: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, 0) + _, err := contentStore.Get(&models.LFSMetaObject{Oid: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, 0) if err == nil { t.Fatalf("expected to get an error, but content existed") } @@ -139,7 +140,7 @@ func TestContenStoreExists(t *testing.T) { setup() defer teardown() - m := &MetaObject{ + m := &models.LFSMetaObject{ Oid: "6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72", Size: 12, } diff --git a/modules/lfs/main.go b/modules/lfs/main.go deleted file mode 100644 index f6069c4d4f05d..0000000000000 --- a/modules/lfs/main.go +++ /dev/null @@ -1,112 +0,0 @@ -package main - -import ( - "crypto/tls" - "fmt" - "net" - "os" - "os/signal" - "syscall" - "time" -) - -const ( - contentMediaType = "application/vnd.git-lfs" - metaMediaType = contentMediaType + "+json" - version = "0.3.0" -) - -var ( - logger = NewKVLogger(os.Stdout) -) - -// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted -// connections. It's used by ListenAndServe and ListenAndServeTLS so -// dead TCP connections (e.g. closing laptop mid-download) eventually -// go away. -type tcpKeepAliveListener struct { - *net.TCPListener -} - -func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) { - tc, err := ln.AcceptTCP() - if err != nil { - return - } - tc.SetKeepAlive(true) - tc.SetKeepAlivePeriod(3 * time.Minute) - return tc, nil -} - -func wrapHttps(l net.Listener, cert, key string) (net.Listener, error) { - var err error - - config := &tls.Config{} - - if config.NextProtos == nil { - config.NextProtos = []string{"http/1.1"} - } - - config.Certificates = make([]tls.Certificate, 1) - config.Certificates[0], err = tls.LoadX509KeyPair(cert, key) - if err != nil { - return nil, err - } - - netListener := l.(*TrackingListener).Listener - - tlsListener := tls.NewListener(tcpKeepAliveListener{netListener.(*net.TCPListener)}, config) - return tlsListener, nil -} - -func main() { - if len(os.Args) == 2 && os.Args[1] == "-v" { - fmt.Println(version) - os.Exit(0) - } - - var listener net.Listener - - tl, err := NewTrackingListener(Config.Listen) - if err != nil { - logger.Fatal(kv{"fn": "main", "err": "Could not create listener: " + err.Error()}) - } - - listener = tl - - if Config.IsHTTPS() { - logger.Log(kv{"fn": "main", "msg": "Using https"}) - listener, err = wrapHttps(tl, Config.Cert, Config.Key) - if err != nil { - logger.Fatal(kv{"fn": "main", "err": "Could not create https listener: " + err.Error()}) - } - } - - metaStore, err := NewMetaStore(Config.MetaDB) - if err != nil { - logger.Fatal(kv{"fn": "main", "err": "Could not open the meta store: " + err.Error()}) - } - - contentStore, err := NewContentStore(Config.ContentPath) - if err != nil { - logger.Fatal(kv{"fn": "main", "err": "Could not open the content store: " + err.Error()}) - } - - c := make(chan os.Signal, 1) - signal.Notify(c, syscall.SIGHUP) - go func(c chan os.Signal, listener net.Listener) { - for { - sig := <-c - switch sig { - case syscall.SIGHUP: // Graceful shutdown - tl.Close() - } - } - }(c, tl) - - logger.Log(kv{"fn": "main", "msg": "listening", "pid": os.Getpid(), "addr": Config.Listen, "version": version}) - - app := NewApp(contentStore, metaStore) - app.Serve(listener) - tl.WaitForChildren() -} diff --git a/modules/lfs/meta_store.go b/modules/lfs/meta_store.go deleted file mode 100644 index 761f873c005cf..0000000000000 --- a/modules/lfs/meta_store.go +++ /dev/null @@ -1,337 +0,0 @@ -package main - -import ( - "bytes" - "encoding/base64" - "encoding/gob" - "errors" - "fmt" - "strings" - "time" - - "github.com/boltdb/bolt" - "github.com/dgrijalva/jwt-go" -) - -// MetaStore implements a metadata storage. It stores user credentials and Meta information -// for objects. The storage is handled by boltdb. -type MetaStore struct { - db *bolt.DB -} - -var ( - errNoBucket = errors.New("Bucket not found") - errObjectNotFound = errors.New("Object not found") -) - -var ( - usersBucket = []byte("users") - objectsBucket = []byte("objects") -) - -// NewMetaStore creates a new MetaStore using the boltdb database at dbFile. -func NewMetaStore(dbFile string) (*MetaStore, error) { - db, err := bolt.Open(dbFile, 0600, &bolt.Options{Timeout: 1 * time.Second}) - if err != nil { - return nil, err - } - - db.Update(func(tx *bolt.Tx) error { - if _, err := tx.CreateBucketIfNotExists(usersBucket); err != nil { - return err - } - - if _, err := tx.CreateBucketIfNotExists(objectsBucket); err != nil { - return err - } - - return nil - }) - - return &MetaStore{db: db}, nil -} - -// Get retrieves the Meta information for an object given information in -// RequestVars -func (s *MetaStore) Get(v *RequestVars) (*MetaObject, error) { - if !s.authenticate(v.Authorization, false) { - return nil, newAuthError() - } - meta, error := s.UnsafeGet(v) - return meta, error -} - -// Get retrieves the Meta information for an object given information in -// RequestVars -// DO NOT CHECK authentication, as it is supposed to have been done before -func (s *MetaStore) UnsafeGet(v *RequestVars) (*MetaObject, error) { - var meta MetaObject - - err := s.db.View(func(tx *bolt.Tx) error { - bucket := tx.Bucket(objectsBucket) - if bucket == nil { - return errNoBucket - } - - value := bucket.Get([]byte(v.Oid)) - if len(value) == 0 { - return errObjectNotFound - } - - dec := gob.NewDecoder(bytes.NewBuffer(value)) - return dec.Decode(&meta) - }) - - if err != nil { - return nil, err - } - - return &meta, nil -} - -// Put writes meta information from RequestVars to the store. -func (s *MetaStore) Put(v *RequestVars) (*MetaObject, error) { - if !s.authenticate(v.Authorization, true) { - return nil, newAuthError() - } - - // Check if it exists first - if meta, err := s.Get(v); err == nil { - meta.Existing = true - return meta, nil - } - - var buf bytes.Buffer - enc := gob.NewEncoder(&buf) - meta := MetaObject{Oid: v.Oid, Size: v.Size} - err := enc.Encode(meta) - if err != nil { - return nil, err - } - - err = s.db.Update(func(tx *bolt.Tx) error { - bucket := tx.Bucket(objectsBucket) - if bucket == nil { - return errNoBucket - } - - err = bucket.Put([]byte(v.Oid), buf.Bytes()) - if err != nil { - return err - } - - return nil - }) - - if err != nil { - return nil, err - } - - return &meta, nil -} - -// Delete removes the meta information from RequestVars to the store. -func (s *MetaStore) Delete(v *RequestVars) error { - if !s.authenticate(v.Authorization, true) { - return newAuthError() - } - - err := s.db.Update(func(tx *bolt.Tx) error { - bucket := tx.Bucket(objectsBucket) - if bucket == nil { - return errNoBucket - } - - err := bucket.Delete([]byte(v.Oid)) - if err != nil { - return err - } - - return nil - }) - - return err -} - -// Close closes the underlying boltdb. -func (s *MetaStore) Close() { - s.db.Close() -} - -// AddUser adds user credentials to the meta store. -func (s *MetaStore) AddUser(user, pass string) error { - err := s.db.Update(func(tx *bolt.Tx) error { - bucket := tx.Bucket(usersBucket) - if bucket == nil { - return errNoBucket - } - - err := bucket.Put([]byte(user), []byte(pass)) - if err != nil { - return err - } - return nil - }) - - return err -} - -// DeleteUser removes user credentials from the meta store. -func (s *MetaStore) DeleteUser(user string) error { - err := s.db.Update(func(tx *bolt.Tx) error { - bucket := tx.Bucket(usersBucket) - if bucket == nil { - return errNoBucket - } - - err := bucket.Delete([]byte(user)) - return err - }) - - return err -} - -// MetaUser encapsulates information about a meta store user -type MetaUser struct { - Name string -} - -// Users returns all MetaUsers in the meta store -func (s *MetaStore) Users() ([]*MetaUser, error) { - var users []*MetaUser - - err := s.db.View(func(tx *bolt.Tx) error { - bucket := tx.Bucket(usersBucket) - if bucket == nil { - return errNoBucket - } - - bucket.ForEach(func(k, v []byte) error { - users = append(users, &MetaUser{string(k)}) - return nil - }) - return nil - }) - - return users, err -} - -// Objects returns all MetaObjects in the meta store -func (s *MetaStore) Objects() ([]*MetaObject, error) { - var objects []*MetaObject - - err := s.db.View(func(tx *bolt.Tx) error { - bucket := tx.Bucket(objectsBucket) - if bucket == nil { - return errNoBucket - } - - bucket.ForEach(func(k, v []byte) error { - var meta MetaObject - dec := gob.NewDecoder(bytes.NewBuffer(v)) - err := dec.Decode(&meta) - if err != nil { - return err - } - objects = append(objects, &meta) - return nil - }) - return nil - }) - - return objects, err -} - -// authenticate uses the authorization string to determine whether -// or not to proceed. This server assumes an HTTP Basic auth format. -func (s *MetaStore) authenticate(authorization string, requireWrite bool) bool { - if Config.IsPublic() { - return true - } - - if authorization == "" { - return false - } - - if s.authenticateToken(authorization, requireWrite) { - return true - } - - if !strings.HasPrefix(authorization, "Basic ") { - return false - } - - c, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(authorization, "Basic ")) - if err != nil { - return false - } - cs := string(c) - i := strings.IndexByte(cs, ':') - if i < 0 { - return false - } - user, password := cs[:i], cs[i+1:] - - // check Basic Authentication (admin) - ok := checkBasicAuth(user, password, true) - if ok { - return true - } - - value := "" - - s.db.View(func(tx *bolt.Tx) error { - bucket := tx.Bucket(usersBucket) - if bucket == nil { - return errNoBucket - } - - value = string(bucket.Get([]byte(user))) - return nil - }) - - if value != "" && value == password { - return true - } - return false -} - -func (s *MetaStore) authenticateToken(authorization string, requireWrite bool) bool { - if !strings.HasPrefix(authorization, "Bearer ") { - return false - } - - token, err := jwt.Parse(authorization[7:], func(t *jwt.Token) (interface{}, error) { - if _, ok := t.Method.(*jwt.SigningMethodHMAC); !ok { - return nil, fmt.Errorf("unexpected signing method: %v", t.Header["alg"]) - } - return Config.DecodeSecret() - }) - if err != nil { - return false - } - claims, claimsOk := token.Claims.(jwt.MapClaims) - if !token.Valid || !claimsOk { - return false - } - - opstr, ok := claims["op"].(string) - if !ok { - return false - } - op := strings.ToLower(strings.TrimSpace(opstr)) - status := op == "upload" || (op == "download" && !requireWrite) - return status -} - -type authError struct { - error -} - -func (e authError) AuthError() bool { - return true -} - -func newAuthError() error { - return authError{errors.New("Forbidden")} -} diff --git a/modules/lfs/meta_store_test.go b/modules/lfs/meta_store_test.go deleted file mode 100644 index 19ae46d23fb7d..0000000000000 --- a/modules/lfs/meta_store_test.go +++ /dev/null @@ -1,111 +0,0 @@ -package main - -import ( - "fmt" - "os" - "testing" -) - -var ( - metaStoreTest *MetaStore -) - -func TestGetWithAuth(t *testing.T) { - setupMeta() - defer teardownMeta() - - meta, err := metaStoreTest.Get(&RequestVars{Authorization: testAuth, Oid: contentOid}) - if err != nil { - t.Fatalf("Error retreiving meta: %s", err) - } - - if meta.Oid != contentOid { - t.Errorf("expected to get content oid, got: %s", meta.Oid) - } - - if meta.Size != contentSize { - t.Errorf("expected to get content size, got: %d", meta.Size) - } -} - -func TestGetWithoutAuth(t *testing.T) { - setupMeta() - defer teardownMeta() - - _, err := metaStoreTest.Get(&RequestVars{Authorization: badAuth, Oid: contentOid}) - if !isAuthError(err) { - t.Errorf("expected auth error, got: %s", err) - } -} - -func TestPutWithAuth(t *testing.T) { - setupMeta() - defer teardownMeta() - - meta, err := metaStoreTest.Put(&RequestVars{Authorization: testAuth, Oid: nonexistingOid, Size: 42}) - if err != nil { - t.Errorf("expected put to succeed, got : %s", err) - } - - if meta.Existing { - t.Errorf("expected meta to not have existed") - } - - meta, err = metaStoreTest.Get(&RequestVars{Authorization: testAuth, Oid: nonexistingOid}) - if err != nil { - t.Errorf("expected to be able to retreive new put, got : %s", err) - } - - if meta.Oid != nonexistingOid { - t.Errorf("expected oids to match, got: %s", meta.Oid) - } - - if meta.Size != 42 { - t.Errorf("expected sizes to match, got: %d", meta.Size) - } - - meta, err = metaStoreTest.Put(&RequestVars{Authorization: testAuth, Oid: nonexistingOid, Size: 42}) - if err != nil { - t.Errorf("expected put to succeed, got : %s", err) - } - - if !meta.Existing { - t.Errorf("expected meta to now exist") - } -} - -func TestPuthWithoutAuth(t *testing.T) { - setupMeta() - defer teardownMeta() - - _, err := metaStoreTest.Put(&RequestVars{Authorization: badAuth, Oid: contentOid, Size: 42}) - if !isAuthError(err) { - t.Errorf("expected auth error, got: %s", err) - } -} - -func setupMeta() { - store, err := NewMetaStore("test-meta-store.db") - if err != nil { - fmt.Printf("error initializing test meta store: %s\n", err) - os.Exit(1) - } - - metaStoreTest = store - if err := metaStoreTest.AddUser(testUser, testPass); err != nil { - teardownMeta() - fmt.Printf("error adding test user to meta store: %s\n", err) - os.Exit(1) - } - - rv := &RequestVars{Authorization: testAuth, Oid: contentOid, Size: contentSize} - if _, err := metaStoreTest.Put(rv); err != nil { - teardownMeta() - fmt.Printf("error seeding test meta store: %s\n", err) - os.Exit(1) - } -} - -func teardownMeta() { - os.RemoveAll("test-meta-store.db") -} diff --git a/modules/lfs/server.go b/modules/lfs/server.go index 1340784a5523d..95d40ae07c08a 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -1,19 +1,27 @@ -package main +package lfs import ( - "crypto/rand" + "encoding/base64" "encoding/json" "fmt" "io" - "net" "net/http" "regexp" "strconv" "strings" "time" - "github.com/gorilla/context" - "github.com/gorilla/mux" + "github.com/dgrijalva/jwt-go" + "github.com/go-gitea/gitea/models" + "github.com/go-gitea/gitea/modules/context" + "github.com/go-gitea/gitea/modules/log" + "github.com/go-gitea/gitea/modules/setting" + "gopkg.in/macaron.v1" +) + +const ( + contentMediaType = "application/vnd.git-lfs" + metaMediaType = contentMediaType + "+json" ) // RequestVars contain variables from the HTTP request. Variables from routing, json body decoding, and @@ -33,13 +41,6 @@ type BatchVars struct { Objects []*RequestVars `json:"objects"` } -// MetaObject is object metadata as seen by the object and metadata stores. -type MetaObject struct { - Oid string `json:"oid"` - Size int64 `json:"size"` - Existing bool -} - type BatchResponse struct { Transfer string `json:"transfer,omitempty"` Objects []*Representation `json:"objects"` @@ -72,11 +73,7 @@ func (v *RequestVars) ObjectLink() string { path += fmt.Sprintf("/objects/%s", v.Oid) - if Config.IsHTTPS() { - return fmt.Sprintf("%s://%s%s", Config.Scheme, Config.Host, path) - } - - return fmt.Sprintf("http://%s%s", Config.Host, path) + return fmt.Sprintf("%slfs%s", setting.AppURL, path) } // link provides a structure used to build a hypermedia representation of an HTTP link. @@ -86,204 +83,199 @@ type link struct { ExpiresAt time.Time `json:"expires_at,omitempty"` } -// App links a Router, ContentStore, and MetaStore to provide the LFS server. -type App struct { - router *mux.Router +type LFSHandler struct { contentStore *ContentStore - metaStore *MetaStore } -// NewApp creates a new App using the ContentStore and MetaStore provided -func NewApp(content *ContentStore, meta *MetaStore) *App { - app := &App{contentStore: content, metaStore: meta} - - r := mux.NewRouter() - - r.HandleFunc("/{user}/{repo}/objects/batch", app.BatchHandler).Methods("POST").MatcherFunc(MetaMatcher) - route := "/{user}/{repo}/objects/{oid}" - r.HandleFunc(route, app.GetContentHandler).Methods("GET", "HEAD").MatcherFunc(ContentMatcher) - r.HandleFunc(route, app.GetMetaHandler).Methods("GET", "HEAD").MatcherFunc(MetaMatcher) - r.HandleFunc(route, app.PutHandler).Methods("PUT").MatcherFunc(ContentMatcher) - - r.HandleFunc("/{user}/{repo}/objects", app.PostHandler).Methods("POST").MatcherFunc(MetaMatcher) - - r.HandleFunc("/objects/batch", app.BatchHandler).Methods("POST").MatcherFunc(MetaMatcher) - route = "/objects/{oid}" - r.HandleFunc(route, app.GetContentHandler).Methods("GET", "HEAD").MatcherFunc(ContentMatcher) - r.HandleFunc(route, app.GetMetaHandler).Methods("GET", "HEAD").MatcherFunc(MetaMatcher) - r.HandleFunc(route, app.PutHandler).Methods("PUT").MatcherFunc(ContentMatcher) +func NewLFSHandler() *LFSHandler { + contentStore, err := NewContentStore(setting.LFS.ContentPath) - r.HandleFunc("/objects", app.PostHandler).Methods("POST").MatcherFunc(MetaMatcher) - - app.addMgmt(r) - - app.router = r + if err != nil { + log.Fatal(4, "Error initializing LFS content store: %s", err) + } + app := &LFSHandler{contentStore: contentStore} return app } -func (a *App) ServeHTTP(w http.ResponseWriter, r *http.Request) { - b := make([]byte, 16) - _, err := rand.Read(b) - if err == nil { - context.Set(r, "RequestID", fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])) - } +func (a *LFSHandler) ObjectOidHandler(ctx *context.Context) { - a.router.ServeHTTP(w, r) -} + if ctx.Req.Method == "GET" || ctx.Req.Method == "HEAD" { + if MetaMatcher(ctx.Req) { + a.GetMetaHandler(ctx) + return + } + if ContentMatcher(ctx.Req) { + a.GetContentHandler(ctx) + return + } + } else if ctx.Req.Method == "PUT" && ContentMatcher(ctx.Req) { + a.PutHandler(ctx) + return + } -// Serve calls http.Serve with the provided Listener and the app's router -func (a *App) Serve(l net.Listener) error { - return http.Serve(l, a) } // GetContentHandler gets the content from the content store -func (a *App) GetContentHandler(w http.ResponseWriter, r *http.Request) { - rv := unpack(r) - meta, err := a.metaStore.Get(rv) +func (a *LFSHandler) GetContentHandler(ctx *context.Context) { + + rv := unpack(ctx) + if !authenticate(rv.Authorization, false) { + requireAuth(ctx) + return + } + + meta, err := models.GetLFSMetaObjectByOid(rv.Oid) if err != nil { - if isAuthError(err) { - requireAuth(w, r) - } else { - writeStatus(w, r, 404) - } + writeStatus(ctx, 404) return } // Support resume download using Range header var fromByte int64 statusCode := 200 - if rangeHdr := r.Header.Get("Range"); rangeHdr != "" { + if rangeHdr := ctx.Req.Header.Get("Range"); rangeHdr != "" { regex := regexp.MustCompile(`bytes=(\d+)\-.*`) match := regex.FindStringSubmatch(rangeHdr) if match != nil && len(match) > 1 { statusCode = 206 fromByte, _ = strconv.ParseInt(match[1], 10, 32) - w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", fromByte, meta.Size-1, int64(meta.Size)-fromByte)) + ctx.Resp.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", fromByte, meta.Size-1, int64(meta.Size)-fromByte)) } } content, err := a.contentStore.Get(meta, fromByte) if err != nil { - writeStatus(w, r, 404) + writeStatus(ctx, 404) return } - w.WriteHeader(statusCode) - io.Copy(w, content) - logRequest(r, statusCode) + ctx.Resp.WriteHeader(statusCode) + io.Copy(ctx.Resp, content) + logRequest(ctx.Req, statusCode) } // GetMetaHandler retrieves metadata about the object -func (a *App) GetMetaHandler(w http.ResponseWriter, r *http.Request) { - rv := unpack(r) - meta, err := a.metaStore.Get(rv) +func (a *LFSHandler) GetMetaHandler(ctx *context.Context) { + + rv := unpack(ctx) + if !authenticate(rv.Authorization, false) { + requireAuth(ctx) + return + } + + meta, err := models.GetLFSMetaObjectByOid(rv.Oid) if err != nil { - if isAuthError(err) { - requireAuth(w, r) - } else { - writeStatus(w, r, 404) - } + writeStatus(ctx, 404) return } - w.Header().Set("Content-Type", metaMediaType) + ctx.Resp.Header().Set("Content-Type", metaMediaType) - if r.Method == "GET" { - enc := json.NewEncoder(w) + if ctx.Req.Method == "GET" { + enc := json.NewEncoder(ctx.Resp) enc.Encode(a.Represent(rv, meta, true, false)) } - logRequest(r, 200) + logRequest(ctx.Req, 200) } // PostHandler instructs the client how to upload data -func (a *App) PostHandler(w http.ResponseWriter, r *http.Request) { - rv := unpack(r) - meta, err := a.metaStore.Put(rv) +func (a *LFSHandler) PostHandler(ctx *context.Context) { + + rv := unpack(ctx) + + if !authenticate(rv.Authorization, true) { + requireAuth(ctx) + } + + meta, err := models.NewLFSMetaObject(&models.LFSMetaObject{Oid: rv.Oid, Size: rv.Size}) + if err != nil { - if isAuthError(err) { - requireAuth(w, r) - } else { - writeStatus(w, r, 404) - } + writeStatus(ctx, 404) return } - w.Header().Set("Content-Type", metaMediaType) + ctx.Resp.Header().Set("Content-Type", metaMediaType) sentStatus := 202 if meta.Existing && a.contentStore.Exists(meta) { sentStatus = 200 } - w.WriteHeader(sentStatus) + ctx.Resp.WriteHeader(sentStatus) - enc := json.NewEncoder(w) + enc := json.NewEncoder(ctx.Resp) enc.Encode(a.Represent(rv, meta, meta.Existing, true)) - logRequest(r, sentStatus) + logRequest(ctx.Req, sentStatus) } // BatchHandler provides the batch api -func (a *App) BatchHandler(w http.ResponseWriter, r *http.Request) { - bv := unpackbatch(r) +func (a *LFSHandler) BatchHandler(ctx *context.Context) { + bv := unpackbatch(ctx) var responseObjects []*Representation // Create a response object for _, object := range bv.Objects { - meta, err := a.metaStore.Get(object) + + if !authenticate(object.Authorization, true) { + requireAuth(ctx) + return + } + + meta, err := models.GetLFSMetaObjectByOid(object.Oid) + if err == nil && a.contentStore.Exists(meta) { // Object is found and exists responseObjects = append(responseObjects, a.Represent(object, meta, true, false)) continue } - if isAuthError(err) { - requireAuth(w, r) - return - } - // Object is not found - meta, err = a.metaStore.Put(object) + meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: object.Oid, Size: object.Size}) + if err == nil { responseObjects = append(responseObjects, a.Represent(object, meta, meta.Existing, true)) } } - w.Header().Set("Content-Type", metaMediaType) + ctx.Resp.Header().Set("Content-Type", metaMediaType) respobj := &BatchResponse{Objects: responseObjects} - enc := json.NewEncoder(w) + enc := json.NewEncoder(ctx.Resp) enc.Encode(respobj) - logRequest(r, 200) + logRequest(ctx.Req, 200) } // PutHandler receives data from the client and puts it into the content store -func (a *App) PutHandler(w http.ResponseWriter, r *http.Request) { - rv := unpack(r) - meta, err := a.metaStore.Get(rv) +func (a *LFSHandler) PutHandler(ctx *context.Context) { + rv := unpack(ctx) + + if !authenticate(rv.Authorization, true) { + requireAuth(ctx) + return + } + + meta, err := models.GetLFSMetaObjectByOid(rv.Oid) + if err != nil { - if isAuthError(err) { - requireAuth(w, r) - } else { - writeStatus(w, r, 404) - } + writeStatus(ctx, 404) return } - if err := a.contentStore.Put(meta, r.Body); err != nil { - a.metaStore.Delete(rv) - w.WriteHeader(500) - fmt.Fprintf(w, `{"message":"%s"}`, err) + if err := a.contentStore.Put(meta, ctx.Req.Body().ReadCloser()); err != nil { + models.RemoveLFSMetaObjectByOid(rv.Oid) + ctx.Resp.WriteHeader(500) + fmt.Fprintf(ctx.Resp, `{"message":"%s"}`, err) return } - logRequest(r, 200) + logRequest(ctx.Req, 200) } // Represent takes a RequestVars and Meta and turns it into a Representation suitable // for json encoding -func (a *App) Represent(rv *RequestVars, meta *MetaObject, download, upload bool) *Representation { +func (a *LFSHandler) Represent(rv *RequestVars, meta *models.LFSMetaObject, download, upload bool) *Representation { rep := &Representation{ Oid: meta.Oid, Size: meta.Size, @@ -292,9 +284,8 @@ func (a *App) Represent(rv *RequestVars, meta *MetaObject, download, upload bool header := make(map[string]string) header["Accept"] = contentMediaType - if !Config.IsPublic() { - header["Authorization"] = rv.Authorization - } + header["Authorization"] = rv.Authorization + if download { rep.Actions["download"] = &link{Href: rv.ObjectLink(), Header: header} } @@ -302,12 +293,13 @@ func (a *App) Represent(rv *RequestVars, meta *MetaObject, download, upload bool if upload { rep.Actions["upload"] = &link{Href: rv.ObjectLink(), Header: header} } + return rep } // ContentMatcher provides a mux.MatcherFunc that only allows requests that contain // an Accept header with the contentMediaType -func ContentMatcher(r *http.Request, m *mux.RouteMatch) bool { +func ContentMatcher(r macaron.Request) bool { mediaParts := strings.Split(r.Header.Get("Accept"), ";") mt := mediaParts[0] return mt == contentMediaType @@ -315,24 +307,24 @@ func ContentMatcher(r *http.Request, m *mux.RouteMatch) bool { // MetaMatcher provides a mux.MatcherFunc that only allows requests that contain // an Accept header with the metaMediaType -func MetaMatcher(r *http.Request, m *mux.RouteMatch) bool { +func MetaMatcher(r macaron.Request) bool { mediaParts := strings.Split(r.Header.Get("Accept"), ";") mt := mediaParts[0] return mt == metaMediaType } -func unpack(r *http.Request) *RequestVars { - vars := mux.Vars(r) +func unpack(ctx *context.Context) *RequestVars { + r := ctx.Req rv := &RequestVars{ - User: vars["user"], - Repo: vars["repo"], - Oid: vars["oid"], + User: ctx.Params("user"), + Repo: ctx.Params("repo"), + Oid: ctx.Params("oid"), Authorization: r.Header.Get("Authorization"), } if r.Method == "POST" { // Maybe also check if +json var p RequestVars - dec := json.NewDecoder(r.Body) + dec := json.NewDecoder(r.Body().ReadCloser()) err := dec.Decode(&p) if err != nil { return rv @@ -346,55 +338,114 @@ func unpack(r *http.Request) *RequestVars { } // TODO cheap hack, unify with unpack -func unpackbatch(r *http.Request) *BatchVars { - vars := mux.Vars(r) +func unpackbatch(ctx *context.Context) *BatchVars { + r := ctx.Req var bv BatchVars - dec := json.NewDecoder(r.Body) + dec := json.NewDecoder(r.Body().ReadCloser()) err := dec.Decode(&bv) if err != nil { return &bv } for i := 0; i < len(bv.Objects); i++ { - bv.Objects[i].User = vars["user"] - bv.Objects[i].Repo = vars["repo"] + bv.Objects[i].User = ctx.Params("user") + bv.Objects[i].Repo = ctx.Params("repo") bv.Objects[i].Authorization = r.Header.Get("Authorization") } return &bv } -func writeStatus(w http.ResponseWriter, r *http.Request, status int) { +func writeStatus(ctx *context.Context, status int) { message := http.StatusText(status) - mediaParts := strings.Split(r.Header.Get("Accept"), ";") + mediaParts := strings.Split(ctx.Req.Header.Get("Accept"), ";") mt := mediaParts[0] if strings.HasSuffix(mt, "+json") { message = `{"message":"` + message + `"}` } - w.WriteHeader(status) - fmt.Fprint(w, message) - logRequest(r, status) + ctx.Resp.WriteHeader(status) + fmt.Fprint(ctx.Resp, message) + logRequest(ctx.Req, status) } -func logRequest(r *http.Request, status int) { - logger.Log(kv{"method": r.Method, "url": r.URL, "status": status, "request_id": context.Get(r, "RequestID")}) +func logRequest(r macaron.Request, status int) { + log.Debug("LFS request - Method: %s, URL: %s, Status %s", r.Method, r.URL, status) } -func isAuthError(err error) bool { - type autherror interface { - AuthError() bool +// authenticate uses the authorization string to determine whether +// or not to proceed. This server assumes an HTTP Basic auth format. +func authenticate(authorization string, requireWrite bool) bool { + + if authorization == "" { + return false + } + + if authenticateToken(authorization, requireWrite) { + return true } - if ae, ok := err.(autherror); ok { - return ae.AuthError() + + if !strings.HasPrefix(authorization, "Basic ") { + return false + } + + c, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(authorization, "Basic ")) + if err != nil { + return false + } + cs := string(c) + i := strings.IndexByte(cs, ':') + if i < 0 { + return false } + user, password := cs[:i], cs[i+1:] + _ = user + _ = password + // TODO check Basic Authentication + return false } -func requireAuth(w http.ResponseWriter, r *http.Request) { - w.Header().Set("WWW-Authenticate", "Basic realm=git-lfs-server") - writeStatus(w, r, 401) +func authenticateToken(authorization string, requireWrite bool) bool { + if !strings.HasPrefix(authorization, "Bearer ") { + return false + } + + token, err := jwt.Parse(authorization[7:], func(t *jwt.Token) (interface{}, error) { + if _, ok := t.Method.(*jwt.SigningMethodHMAC); !ok { + return nil, fmt.Errorf("unexpected signing method: %v", t.Header["alg"]) + } + return setting.LFS.JWTSecretBytes, nil + }) + if err != nil { + return false + } + claims, claimsOk := token.Claims.(jwt.MapClaims) + if !token.Valid || !claimsOk { + return false + } + + opstr, ok := claims["op"].(string) + if !ok { + return false + } + op := strings.ToLower(strings.TrimSpace(opstr)) + status := op == "upload" || (op == "download" && !requireWrite) + return status +} + +type authError struct { + error +} + +func (e authError) AuthError() bool { + return true +} + +func requireAuth(ctx *context.Context) { + ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gogs-lfs") + writeStatus(ctx, 401) } diff --git a/modules/lfs/server_test.go b/modules/lfs/server_test.go deleted file mode 100644 index 4a235192d28ad..0000000000000 --- a/modules/lfs/server_test.go +++ /dev/null @@ -1,406 +0,0 @@ -package main - -import ( - "bytes" - "encoding/base64" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "net/http/httptest" - "os" - "testing" -) - -func TestGetAuthed(t *testing.T) { - req, err := http.NewRequest("GET", lfsServer.URL+"/user/repo/objects/"+contentOid, nil) - if err != nil { - t.Fatalf("request error: %s", err) - } - req.SetBasicAuth(testUser, testPass) - req.Header.Set("Accept", contentMediaType) - - res, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("response error: %s", err) - } - - if res.StatusCode != 200 { - t.Fatalf("expected status 302, got %d", res.StatusCode) - } - - by, err := ioutil.ReadAll(res.Body) - if err != nil { - t.Fatalf("expected response to contain content, got error: %s", err) - } - - if string(by) != content { - t.Fatalf("expected content to be `content`, got: %s", string(by)) - } -} - -func TestGetAuthedWithRange(t *testing.T) { - req, err := http.NewRequest("GET", lfsServer.URL+"/user/repo/objects/"+contentOid, nil) - if err != nil { - t.Fatalf("request error: %s", err) - } - req.SetBasicAuth(testUser, testPass) - req.Header.Set("Accept", contentMediaType) - fromByte := 5 - req.Header.Set("Range", fmt.Sprintf("bytes=%d-", fromByte)) - - res, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("response error: %s", err) - } - - if res.StatusCode != 206 { - t.Fatalf("expected status 206, got %d", res.StatusCode) - } - if cr := res.Header.Get("Content-Range"); len(cr) > 0 { - expected := fmt.Sprintf("bytes %d-%d/%d", fromByte, len(content)-1, len(content)-fromByte) - if cr != expected { - t.Fatalf("expected Content-Range header of %q, got %q", expected, cr) - } - } else { - t.Fatalf("missing Content-Range header in response") - } - - by, err := ioutil.ReadAll(res.Body) - if err != nil { - t.Fatalf("expected response to contain content, got error: %s", err) - } - - if string(by) != content[fromByte:] { - t.Fatalf("expected content to be `content`, got: %s", string(by)) - } -} - -func TestGetUnauthed(t *testing.T) { - req, err := http.NewRequest("GET", lfsServer.URL+"/user/repo/objects/"+contentOid, nil) - if err != nil { - t.Fatalf("request error: %s", err) - } - req.Header.Set("Accept", contentMediaType) - - res, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("response error: %s", err) - } - - if res.StatusCode != 401 { - t.Fatalf("expected status 401, got %d %s", res.StatusCode, req.URL) - } -} - -func TestGetMetaAuthed(t *testing.T) { - req, err := http.NewRequest("GET", lfsServer.URL+"/bilbo/repo/objects/"+contentOid, nil) - if err != nil { - t.Fatalf("request error: %s", err) - } - req.SetBasicAuth(testUser, testPass) - req.Header.Set("Accept", metaMediaType) - - res, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("response error: %s", err) - } - - if res.StatusCode != 200 { - t.Fatalf("expected status 200, got %d %s", res.StatusCode, req.URL) - } - - var meta Representation - dec := json.NewDecoder(res.Body) - dec.Decode(&meta) - - if meta.Oid != contentOid { - t.Fatalf("expected to see oid `%s` in meta, got: `%s`", contentOid, meta.Oid) - } - - if meta.Size != contentSize { - t.Fatalf("expected to see a size of `%d`, got: `%d`", contentSize, meta.Size) - } - - download := meta.Actions["download"] - if download.Href != "http://localhost:8080/bilbo/repo/objects/"+contentOid { - t.Fatalf("expected download link, got %s", download.Href) - } -} - -func TestGetMetaUnauthed(t *testing.T) { - req, err := http.NewRequest("GET", lfsServer.URL+"/user/repo/objects/"+contentOid, nil) - if err != nil { - t.Fatalf("request error: %s", err) - } - req.Header.Set("Accept", metaMediaType) - - res, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("response error: %s", err) - } - - if res.StatusCode != 401 { - t.Fatalf("expected status 401, got %d", res.StatusCode) - } -} - -func TestPostAuthedNewObject(t *testing.T) { - req, err := http.NewRequest("POST", lfsServer.URL+"/bilbo/repo/objects", nil) - if err != nil { - t.Fatalf("request error: %s", err) - } - req.SetBasicAuth(testUser, testPass) - req.Header.Set("Accept", metaMediaType) - - buf := bytes.NewBufferString(fmt.Sprintf(`{"oid":"%s", "size":1234}`, nonexistingOid)) - req.Body = ioutil.NopCloser(buf) - - res, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("response error: %s", err) - } - - if res.StatusCode != 202 { - t.Fatalf("expected status 202, got %d", res.StatusCode) - } - - var meta Representation - dec := json.NewDecoder(res.Body) - dec.Decode(&meta) - - if meta.Oid != nonexistingOid { - t.Fatalf("expected to see oid `%s` in meta, got: `%s`", nonexistingOid, meta.Oid) - } - - if meta.Size != 1234 { - t.Fatalf("expected to see a size of `1234`, got: `%d`", meta.Size) - } - - if download, ok := meta.Actions["download"]; ok { - t.Fatalf("expected POST to not contain a download link, got %s", download.Href) - } - - upload, ok := meta.Actions["upload"] - if !ok { - t.Fatal("expected upload link to be present") - } - - if upload.Href != "http://localhost:8080/bilbo/repo/objects/"+nonexistingOid { - t.Fatalf("expected upload link, got %s", upload.Href) - } -} - -func TestPostAuthedExistingObject(t *testing.T) { - req, err := http.NewRequest("POST", lfsServer.URL+"/bilbo/repo/objects", nil) - if err != nil { - t.Fatalf("request error: %s", err) - } - req.SetBasicAuth(testUser, testPass) - req.Header.Set("Accept", metaMediaType) - - buf := bytes.NewBufferString(fmt.Sprintf(`{"oid":"%s", "size":%d}`, contentOid, contentSize)) - req.Body = ioutil.NopCloser(buf) - - res, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("response error: %s", err) - } - - if res.StatusCode != 200 { - t.Fatalf("expected status 200, got %d", res.StatusCode) - } - - var meta Representation - dec := json.NewDecoder(res.Body) - dec.Decode(&meta) - - if meta.Oid != contentOid { - t.Fatalf("expected to see oid `%s` in meta, got: `%s`", contentOid, meta.Oid) - } - - if meta.Size != contentSize { - t.Fatalf("expected to see a size of `%d`, got: `%d`", contentSize, meta.Size) - } - - download := meta.Actions["download"] - if download.Href != "http://localhost:8080/bilbo/repo/objects/"+contentOid { - t.Fatalf("expected download link, got %s", download.Href) - } - - upload, ok := meta.Actions["upload"] - if !ok { - t.Fatalf("expected upload link to be present") - } - - if upload.Href != "http://localhost:8080/bilbo/repo/objects/"+contentOid { - t.Fatalf("expected upload link, got %s", upload.Href) - } -} - -func TestPostUnauthed(t *testing.T) { - req, err := http.NewRequest("POST", lfsServer.URL+"/bilbo/readonly/objects", nil) - if err != nil { - t.Fatalf("request error: %s", err) - } - req.Header.Set("Accept", metaMediaType) - - buf := bytes.NewBufferString(fmt.Sprintf(`{"oid":"%s", "size":%d}`, contentOid, contentSize)) - req.Body = ioutil.NopCloser(buf) - - res, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("response error: %s", err) - } - - if res.StatusCode != 401 { - t.Fatalf("expected status 401, got %d", res.StatusCode) - } -} - -func TestPut(t *testing.T) { - req, err := http.NewRequest("PUT", lfsServer.URL+"/user/repo/objects/"+contentOid, nil) - if err != nil { - t.Fatalf("request error: %s", err) - } - req.SetBasicAuth(testUser, testPass) - req.Header.Set("Accept", contentMediaType) - req.Header.Set("Content-Type", "application/octet-stream") - req.Body = ioutil.NopCloser(bytes.NewBuffer([]byte(content))) - - res, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("response error: %s", err) - } - - if res.StatusCode != 200 { - t.Fatalf("expected status 200, got %d", res.StatusCode) - } - - r, err := testContentStore.Get(&MetaObject{Oid: contentOid}, 0) - if err != nil { - t.Fatalf("error retreiving from content store: %s", err) - } - c, err := ioutil.ReadAll(r) - if err != nil { - t.Fatalf("error reading content: %s", err) - } - if string(c) != content { - t.Fatalf("expected content, got `%s`", string(c)) - } -} - -func TestMediaTypesRequired(t *testing.T) { - m := []string{"GET", "PUT", "POST", "HEAD"} - for _, method := range m { - req, err := http.NewRequest(method, lfsServer.URL+"/user/repo/objects/"+contentOid, nil) - if err != nil { - t.Fatalf("request error: %s", err) - } - req.SetBasicAuth(testUser, testPass) - res, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("response error: %s", err) - } - - if res.StatusCode != 404 { - t.Fatalf("expected status 404, got %d", res.StatusCode) - } - } -} - -func TestMediaTypesParsed(t *testing.T) { - req, err := http.NewRequest("GET", lfsServer.URL+"/user/repo/objects/"+contentOid, nil) - if err != nil { - t.Fatalf("request error: %s", err) - } - req.SetBasicAuth(testUser, testPass) - req.Header.Set("Accept", contentMediaType+"; charset=utf-8") - - res, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("response error: %s", err) - } - - if res.StatusCode != 200 { - t.Fatalf("expected status 200, got %d", res.StatusCode) - } -} - -var ( - lfsServer *httptest.Server - testMetaStore *MetaStore - testContentStore *ContentStore - testUser = "bilbo" - testPass = "baggins" - testAuth = fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(testUser+":"+testPass))) - badAuth = fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte("azog:defiler"))) - content = "this is my content" - contentSize = int64(len(content)) - contentOid = "f97e1b2936a56511b3b6efc99011758e4700d60fb1674d31445d1ee40b663f24" - nonexistingOid = "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f" -) - -func TestMain(m *testing.M) { - os.Remove("lfs-test.db") - - var err error - testMetaStore, err = NewMetaStore("lfs-test.db") - if err != nil { - fmt.Printf("Error creating meta store: %s", err) - os.Exit(1) - } - - testContentStore, err = NewContentStore("lfs-content-test") - if err != nil { - fmt.Printf("Error creating content store: %s", err) - os.Exit(1) - } - - if err := seedMetaStore(); err != nil { - fmt.Printf("Error seeding meta store: %s", err) - os.Exit(1) - } - - if err := seedContentStore(); err != nil { - fmt.Printf("Error seeding content store: %s", err) - os.Exit(1) - } - - app := NewApp(testContentStore, testMetaStore) - lfsServer = httptest.NewServer(app) - - logger = NewKVLogger(ioutil.Discard) - - ret := m.Run() - - lfsServer.Close() - testMetaStore.Close() - os.Remove("lfs-test.db") - os.RemoveAll("lfs-content-test") - - os.Exit(ret) -} - -func seedMetaStore() error { - if err := testMetaStore.AddUser(testUser, testPass); err != nil { - return err - } - - rv := &RequestVars{Authorization: testAuth, Oid: contentOid, Size: contentSize} - if _, err := testMetaStore.Put(rv); err != nil { - return err - } - - return nil -} - -func seedContentStore() error { - meta := &MetaObject{Oid: contentOid, Size: contentSize} - buf := bytes.NewBuffer([]byte(content)) - if err := testContentStore.Put(meta, buf); err != nil { - return err - } - - return nil -} diff --git a/modules/lfs/tracking_listener.go b/modules/lfs/tracking_listener.go deleted file mode 100644 index bd76cd4081a2f..0000000000000 --- a/modules/lfs/tracking_listener.go +++ /dev/null @@ -1,95 +0,0 @@ -package main - -import ( - "fmt" - "net" - "net/url" - "os" - "strconv" - "sync" -) - -// TrackingListener tracks incoming connections so that application shutdown can -// wait until all in progress connections are finished before exiting. -type TrackingListener struct { - wg sync.WaitGroup - connections map[net.Conn]bool - cm sync.Mutex - net.Listener -} - -// NewTrackingListener creates a new TrackingListener, listening on the supplied -// address. -func NewTrackingListener(addr string) (*TrackingListener, error) { - var listener net.Listener - - a, err := url.Parse(addr) - if err != nil { - return nil, err - } - - switch a.Scheme { - case "fd": - fd, err := strconv.Atoi(a.Host) - if err != nil { - return nil, err - } - - f := os.NewFile(uintptr(fd), "trackinglistener") - listener, err = net.FileListener(f) - if err != nil { - return nil, err - } - case "tcp", "tcp4", "tcp6": - laddr, err := net.ResolveTCPAddr(a.Scheme, a.Host) - if err != nil { - return nil, err - } - - listener, err = net.ListenTCP(a.Scheme, laddr) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("Unsupported listener protocol: %s", a.Scheme) - } - - return &TrackingListener{Listener: listener, connections: make(map[net.Conn]bool)}, nil -} - -// Accept wraps the underlying net.Listener's Accept(), keeping track of all connections -// accepted. -func (l *TrackingListener) Accept() (net.Conn, error) { - l.wg.Add(1) - conn, err := l.Listener.Accept() - if err != nil { - l.wg.Done() - return nil, err - } - - c := &trackedConn{ - Conn: conn, - listener: l, - } - - return c, nil -} - -// WaitForChildren is called during shutdown. It will return once all the existing -// connections have finished. -func (l *TrackingListener) WaitForChildren() { - l.wg.Wait() - logger.Log(kv{"fn": "shutdown"}) -} - -type trackedConn struct { - net.Conn - listener *TrackingListener - once sync.Once -} - -func (c *trackedConn) Close() error { - c.once.Do(c.listener.wg.Done) - - return c.Conn.Close() -} From f1118113fe7fc6cb878409026f2e74b6b27c8e47 Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 7 Nov 2016 14:40:15 +0100 Subject: [PATCH 08/37] Move LFS routes to /user/repo/info/lfs/* --- cmd/web.go | 33 ++++++++++--------------------- models/lfs.go | 12 +++++------ modules/lfs/server.go | 46 ++++++++++++++++++++++++------------------- 3 files changed, 42 insertions(+), 49 deletions(-) diff --git a/cmd/web.go b/cmd/web.go index 4d0b002c835ba..88eb1dce2b723 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -565,35 +565,22 @@ func runWeb(ctx *cli.Context) error { }, ignSignIn, context.RepoAssignment(true), context.RepoRef()) m.Group("/:reponame", func() { + + if setting.LFS.StartServer { + lfsHandler := lfs.NewLFSHandler() + m.Group("/info/lfs", func() { + m.Post("/objects/batch", lfsHandler.BatchHandler) // TODO MetaMatcher + m.Any("/objects/:oid", lfsHandler.ObjectOidHandler) + m.Post("/objects", lfsHandler.PostHandler) // TODO MetaMatcher + }, ignSignInAndCsrf) + } + m.Any("/*", ignSignInAndCsrf, repo.HTTP) m.Head("/tasks/trigger", repo.TriggerTask) }) }) // ***** END: Repository ***** - // ***** START: LFS ***** - - if setting.LFS.StartServer { - - lfsHandler := lfs.NewLFSHandler() - - m.Group("/lfs", func() { - - m.Post("/:user/:repo/objects/batch", lfsHandler.BatchHandler) // TODO MetaMatcher - m.Any("/:user/:repo/objects/:oid", lfsHandler.ObjectOidHandler) - - m.Post("/:user/:repo/objects", lfsHandler.PostHandler) // TODO MetaMatcher - m.Post("/objects/batch", lfsHandler.BatchHandler) // TODO MetaMatcher - - m.Any("/objects/:oid", lfsHandler.ObjectOidHandler) - m.Post("/objects", lfsHandler.PostHandler) // TODO MetaMatcher - - }, ignSignInAndCsrf) - - } - - // ***** END: LFS ***** - m.Group("/api", func() { apiv1.RegisterRoutes(m) }, ignSignIn) diff --git a/models/lfs.go b/models/lfs.go index 592daabd584ed..24171fcf1436a 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -7,12 +7,12 @@ import ( ) type LFSMetaObject struct { - ID int64 `xorm:"pk autoincr"` - Oid string `xorm:"UNIQUE NOT NULL"` - Size int64 `xorm:"NOT NULL"` - Existing bool `xorm:"-"` - Created time.Time `xorm:"-"` - CreatedUnix int64 + Oid string `xorm:"pk"` + Size int64 `xorm:"NOT NULL"` + RepositoryID int64 `xorm:"NOT NULL"` + Existing bool `xorm:"-"` + Created time.Time `xorm:"-"` + CreatedUnix int64 } var ( diff --git a/modules/lfs/server.go b/modules/lfs/server.go index 95d40ae07c08a..f55ade27b3506 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -61,19 +61,7 @@ type ObjectError struct { // ObjectLink builds a URL linking to the object. func (v *RequestVars) ObjectLink() string { - path := "" - - if len(v.User) > 0 { - path += fmt.Sprintf("/%s", v.User) - } - - if len(v.Repo) > 0 { - path += fmt.Sprintf("/%s", v.Repo) - } - - path += fmt.Sprintf("/objects/%s", v.Oid) - - return fmt.Sprintf("%slfs%s", setting.AppURL, path) + return fmt.Sprintf("%s%s/%s/info/lfs/objects/%s", setting.AppURL, v.User, v.Repo, v.Oid) } // link provides a structure used to build a hypermedia representation of an HTTP link. @@ -189,7 +177,16 @@ func (a *LFSHandler) PostHandler(ctx *context.Context) { requireAuth(ctx) } - meta, err := models.NewLFSMetaObject(&models.LFSMetaObject{Oid: rv.Oid, Size: rv.Size}) + repositoryString := rv.User + "/" + rv.Repo + repository, err := models.GetRepositoryByRef(repositoryString) + + if err != nil { + log.Debug("Could not find repository: %s - %s", repositoryString, err) + writeStatus(ctx, 404) + return + } + + meta, err := models.NewLFSMetaObject(&models.LFSMetaObject{Oid: rv.Oid, Size: rv.Size, RepositoryID: repository.ID}) if err != nil { writeStatus(ctx, 404) @@ -230,8 +227,17 @@ func (a *LFSHandler) BatchHandler(ctx *context.Context) { continue } + repositoryString := object.User + "/" + object.Repo + repository, err := models.GetRepositoryByRef(repositoryString) + + if err != nil { + log.Debug("Could not find repository: %s - %s", repositoryString, err) + writeStatus(ctx, 404) + return + } + // Object is not found - meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: object.Oid, Size: object.Size}) + meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: object.Oid, Size: object.Size, RepositoryID: repository.ID}) if err == nil { responseObjects = append(responseObjects, a.Represent(object, meta, meta.Existing, true)) @@ -316,8 +322,8 @@ func MetaMatcher(r macaron.Request) bool { func unpack(ctx *context.Context) *RequestVars { r := ctx.Req rv := &RequestVars{ - User: ctx.Params("user"), - Repo: ctx.Params("repo"), + User: ctx.Params("username"), + Repo: strings.TrimSuffix(ctx.Params("reponame"), ".git"), Oid: ctx.Params("oid"), Authorization: r.Header.Get("Authorization"), } @@ -350,8 +356,8 @@ func unpackbatch(ctx *context.Context) *BatchVars { } for i := 0; i < len(bv.Objects); i++ { - bv.Objects[i].User = ctx.Params("user") - bv.Objects[i].Repo = ctx.Params("repo") + bv.Objects[i].User = ctx.Params("username") + bv.Objects[i].Repo = strings.TrimSuffix(ctx.Params("reponame"), ".git") bv.Objects[i].Authorization = r.Header.Get("Authorization") } @@ -373,7 +379,7 @@ func writeStatus(ctx *context.Context, status int) { } func logRequest(r macaron.Request, status int) { - log.Debug("LFS request - Method: %s, URL: %s, Status %s", r.Method, r.URL, status) + log.Debug("LFS request - Method: %s, URL: %s, Status %d", r.Method, r.URL, status) } // authenticate uses the authorization string to determine whether From c190661ab3b7dbccf40acbb8097f25a23a15e58d Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 7 Nov 2016 14:44:11 +0100 Subject: [PATCH 09/37] Add request header checks to LFS BatchHandler / PostHandler --- cmd/web.go | 4 ++-- modules/lfs/server.go | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/cmd/web.go b/cmd/web.go index 88eb1dce2b723..8785c6c4caa22 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -569,9 +569,9 @@ func runWeb(ctx *cli.Context) error { if setting.LFS.StartServer { lfsHandler := lfs.NewLFSHandler() m.Group("/info/lfs", func() { - m.Post("/objects/batch", lfsHandler.BatchHandler) // TODO MetaMatcher + m.Post("/objects/batch", lfsHandler.BatchHandler) m.Any("/objects/:oid", lfsHandler.ObjectOidHandler) - m.Post("/objects", lfsHandler.PostHandler) // TODO MetaMatcher + m.Post("/objects", lfsHandler.PostHandler) }, ignSignInAndCsrf) } diff --git a/modules/lfs/server.go b/modules/lfs/server.go index f55ade27b3506..8dd66d2f479ca 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -171,6 +171,11 @@ func (a *LFSHandler) GetMetaHandler(ctx *context.Context) { // PostHandler instructs the client how to upload data func (a *LFSHandler) PostHandler(ctx *context.Context) { + if !MetaMatcher(ctx.Req) { + writeStatus(ctx, 500) + return + } + rv := unpack(ctx) if !authenticate(rv.Authorization, true) { @@ -208,6 +213,12 @@ func (a *LFSHandler) PostHandler(ctx *context.Context) { // BatchHandler provides the batch api func (a *LFSHandler) BatchHandler(ctx *context.Context) { + + if !MetaMatcher(ctx.Req) { + writeStatus(ctx, 500) + return + } + bv := unpackbatch(ctx) var responseObjects []*Representation From 73c52edde55774c96301026a651384001c6a6ce0 Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 7 Nov 2016 15:00:40 +0100 Subject: [PATCH 10/37] Implement LFS basic authentication --- modules/lfs/server.go | 105 +++++++++++++++++++++++++++++------------- 1 file changed, 73 insertions(+), 32 deletions(-) diff --git a/modules/lfs/server.go b/modules/lfs/server.go index 8dd66d2f479ca..5b95ecb381f17 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -108,17 +108,25 @@ func (a *LFSHandler) ObjectOidHandler(ctx *context.Context) { func (a *LFSHandler) GetContentHandler(ctx *context.Context) { rv := unpack(ctx) - if !authenticate(rv.Authorization, false) { - requireAuth(ctx) + + meta, err := models.GetLFSMetaObjectByOid(rv.Oid) + if err != nil { + writeStatus(ctx, 404) return } - meta, err := models.GetLFSMetaObjectByOid(rv.Oid) + repository, err := models.GetRepositoryByID(meta.RepositoryID) + if err != nil { writeStatus(ctx, 404) return } + if !authenticate(repository, rv.Authorization, false) { + requireAuth(ctx) + return + } + // Support resume download using Range header var fromByte int64 statusCode := 200 @@ -147,17 +155,25 @@ func (a *LFSHandler) GetContentHandler(ctx *context.Context) { func (a *LFSHandler) GetMetaHandler(ctx *context.Context) { rv := unpack(ctx) - if !authenticate(rv.Authorization, false) { - requireAuth(ctx) + + meta, err := models.GetLFSMetaObjectByOid(rv.Oid) + if err != nil { + writeStatus(ctx, 404) return } - meta, err := models.GetLFSMetaObjectByOid(rv.Oid) + repository, err := models.GetRepositoryByID(meta.RepositoryID) + if err != nil { writeStatus(ctx, 404) return } + if !authenticate(repository, rv.Authorization, false) { + requireAuth(ctx) + return + } + ctx.Resp.Header().Set("Content-Type", metaMediaType) if ctx.Req.Method == "GET" { @@ -178,10 +194,6 @@ func (a *LFSHandler) PostHandler(ctx *context.Context) { rv := unpack(ctx) - if !authenticate(rv.Authorization, true) { - requireAuth(ctx) - } - repositoryString := rv.User + "/" + rv.Repo repository, err := models.GetRepositoryByRef(repositoryString) @@ -191,6 +203,10 @@ func (a *LFSHandler) PostHandler(ctx *context.Context) { return } + if !authenticate(repository, rv.Authorization, true) { + requireAuth(ctx) + } + meta, err := models.NewLFSMetaObject(&models.LFSMetaObject{Oid: rv.Oid, Size: rv.Size, RepositoryID: repository.ID}) if err != nil { @@ -226,7 +242,16 @@ func (a *LFSHandler) BatchHandler(ctx *context.Context) { // Create a response object for _, object := range bv.Objects { - if !authenticate(object.Authorization, true) { + repositoryString := object.User + "/" + object.Repo + repository, err := models.GetRepositoryByRef(repositoryString) + + if err != nil { + log.Debug("Could not find repository: %s - %s", repositoryString, err) + writeStatus(ctx, 404) + return + } + + if !authenticate(repository, object.Authorization, true) { requireAuth(ctx) return } @@ -238,15 +263,6 @@ func (a *LFSHandler) BatchHandler(ctx *context.Context) { continue } - repositoryString := object.User + "/" + object.Repo - repository, err := models.GetRepositoryByRef(repositoryString) - - if err != nil { - log.Debug("Could not find repository: %s - %s", repositoryString, err) - writeStatus(ctx, 404) - return - } - // Object is not found meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: object.Oid, Size: object.Size, RepositoryID: repository.ID}) @@ -268,18 +284,25 @@ func (a *LFSHandler) BatchHandler(ctx *context.Context) { func (a *LFSHandler) PutHandler(ctx *context.Context) { rv := unpack(ctx) - if !authenticate(rv.Authorization, true) { - requireAuth(ctx) + meta, err := models.GetLFSMetaObjectByOid(rv.Oid) + + if err != nil { + writeStatus(ctx, 404) return } - meta, err := models.GetLFSMetaObjectByOid(rv.Oid) + repository, err := models.GetRepositoryByID(meta.RepositoryID) if err != nil { writeStatus(ctx, 404) return } + if !authenticate(repository, rv.Authorization, true) { + requireAuth(ctx) + return + } + if err := a.contentStore.Put(meta, ctx.Req.Body().ReadCloser()); err != nil { models.RemoveLFSMetaObjectByOid(rv.Oid) ctx.Resp.WriteHeader(500) @@ -395,15 +418,19 @@ func logRequest(r macaron.Request, status int) { // authenticate uses the authorization string to determine whether // or not to proceed. This server assumes an HTTP Basic auth format. -func authenticate(authorization string, requireWrite bool) bool { +func authenticate(repository *models.Repository, authorization string, requireWrite bool) bool { + + if !repository.IsPrivate && !requireWrite { + return true + } if authorization == "" { return false } - if authenticateToken(authorization, requireWrite) { - return true - } + //if authenticateToken(authorization, requireWrite) { + // return true + //} if !strings.HasPrefix(authorization, "Basic ") { return false @@ -419,14 +446,27 @@ func authenticate(authorization string, requireWrite bool) bool { return false } user, password := cs[:i], cs[i+1:] - _ = user - _ = password - // TODO check Basic Authentication - return false + userModel, err := models.GetUserByName(user) + if err != nil { + return false + } + + if !userModel.ValidatePassword(password) { + return false + } + + accessMode := models.AccessModeRead + if requireWrite { + accessMode = models.AccessModeWrite + } + + accessCheck, _ := models.HasAccess(userModel, repository, accessMode) + return accessCheck } -func authenticateToken(authorization string, requireWrite bool) bool { +/* +func authenticateToken(repository *models.Repository, authorization string, requireWrite bool) bool { if !strings.HasPrefix(authorization, "Bearer ") { return false } @@ -453,6 +493,7 @@ func authenticateToken(authorization string, requireWrite bool) bool { status := op == "upload" || (op == "download" && !requireWrite) return status } +*/ type authError struct { error From 3b78c5b7762a7ba847fdbcbe6efb36bc8b3ad2d1 Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 7 Nov 2016 16:15:37 +0100 Subject: [PATCH 11/37] Rework JWT secret generation / load --- modules/base/tool.go | 13 +++++++++++++ modules/setting/setting.go | 29 +++++++++++++++++++++++++---- routers/install.go | 3 +++ 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/modules/base/tool.go b/modules/base/tool.go index cb2eafd30990b..1f02b73bbeb24 100644 --- a/modules/base/tool.go +++ b/modules/base/tool.go @@ -12,6 +12,7 @@ import ( "encoding/hex" "fmt" "html/template" + "io" "math" "math/big" "net/http" @@ -103,6 +104,18 @@ func GetRandomString(n int) (string, error) { return string(buffer), nil } +// GetRandomBytes generates a random base64 string from n bytes +func GetRandomBytesAsBase64(n int) string { + bytes := make([]byte, 32) + _, err := io.ReadFull(rand.Reader, bytes) + + if err != nil { + log.Fatal(4, "Error reading random bytes: %s", err) + } + + return base64.RawURLEncoding.EncodeToString(bytes) +} + func randomInt(max *big.Int) (int, error) { rand, err := rand.Int(rand.Reader, max) if err != nil { diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 176f59f8ab1cf..66750b28e81aa 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -603,13 +603,34 @@ please consider changing to GITEA_CUSTOM`) } LFS.JWTSecretBytes = make([]byte, 32) - n, err := base64.StdEncoding.Decode(LFS.JWTSecretBytes, []byte(LFS.JWTSecretBase64)) + n, err := base64.RawURLEncoding.Decode(LFS.JWTSecretBytes, []byte(LFS.JWTSecretBase64)) if err != nil || n != 32 { - log.Warn("Failed loading LFS JWT secret, generating runtime secret key") - _, err = io.ReadFull(rand.Reader, LFS.JWTSecretBytes) + //Generate new secret and save to config + + _, err := io.ReadFull(rand.Reader, LFS.JWTSecretBytes) + if err != nil { - log.Fatal(4, "Error generating temporary JWT secret for LFS server: %v", err) + log.Fatal(4, "Error reading random bytes: %s", err) + } + + LFS.JWTSecretBase64 = base64.RawURLEncoding.EncodeToString(LFS.JWTSecretBytes) + + // Save secret + cfg := ini.Empty() + if com.IsFile(CustomConf) { + // Keeps custom settings if there is already something. + if err := cfg.Append(CustomConf); err != nil { + log.Error(4, "Fail to load custom conf '%s': %v", CustomConf, err) + } + } + + cfg.Section("server").Key("LFS_JWT_SECRET").SetValue(LFS.JWTSecretBase64) + + os.MkdirAll(filepath.Dir(CustomConf), os.ModePerm) + if err := cfg.SaveTo(CustomConf); err != nil { + log.Fatal(4, "Error saving generated JWT Secret to custom config: %v", err) + return } } } diff --git a/routers/install.go b/routers/install.go index 9553f77b35e58..78e9da0454d2a 100644 --- a/routers/install.go +++ b/routers/install.go @@ -254,6 +254,9 @@ func InstallPost(ctx *context.Context, form auth.InstallForm) { cfg.Section("server").Key("SSH_PORT").SetValue(com.ToStr(form.SSHPort)) } + //TODO Expose LFS settings in install form + cfg.Section("server").Key("LFS_JWT_SECRET").SetValue(base.GetRandomBytesAsBase64(32)) + if len(strings.TrimSpace(form.SMTPHost)) > 0 { cfg.Section("mailer").Key("ENABLED").SetValue("true") cfg.Section("mailer").Key("HOST").SetValue(form.SMTPHost) From f7c78d93de36de4e895fed67c134b0b0bc33b366 Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Mon, 7 Nov 2016 19:34:45 +0100 Subject: [PATCH 12/37] Implement LFS SSH token authentication with JWT Specification: https://github.com/github/git-lfs/tree/master/docs/api --- cmd/serve.go | 56 ++++++++++++++++++++++++++++++++++++++++++- models/lfs.go | 5 ++++ modules/lfs/server.go | 35 +++++++++++++++++++-------- 3 files changed, 85 insertions(+), 11 deletions(-) diff --git a/cmd/serve.go b/cmd/serve.go index f806db2096238..552add5035247 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -7,6 +7,7 @@ package cmd import ( "crypto/tls" + "encoding/json" "fmt" "os" "os/exec" @@ -21,12 +22,14 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "github.com/Unknwon/com" + "github.com/dgrijalva/jwt-go" gouuid "github.com/satori/go.uuid" "github.com/urfave/cli" ) const ( - accessDenied = "Repository does not exist or you do not have access" + accessDenied = "Repository does not exist or you do not have access" + lfsAuthenticateVerb = "git-lfs-authenticate" ) // CmdServ represents the available serv sub-command. @@ -73,6 +76,7 @@ var ( "git-upload-pack": models.AccessModeRead, "git-upload-archive": models.AccessModeRead, "git-receive-pack": models.AccessModeWrite, + lfsAuthenticateVerb: models.AccessModeNone, } ) @@ -161,6 +165,14 @@ func runServ(c *cli.Context) error { } verb, args := parseCmd(cmd) + + var lfsVerb string + if verb == lfsAuthenticateVerb && strings.Contains(args, " ") { + argsSplit := strings.SplitN(args, " ", 2) + args = strings.TrimSpace(argsSplit[0]) + lfsVerb = strings.TrimSpace(argsSplit[1]) + } + repoPath := strings.ToLower(strings.Trim(args, "'")) rr := strings.SplitN(repoPath, "/", 2) if len(rr) != 2 { @@ -196,6 +208,14 @@ func runServ(c *cli.Context) error { fail("Unknown git command", "Unknown git command %s", verb) } + if verb == lfsAuthenticateVerb { + if lfsVerb == "upload" { + requestedMode = models.AccessModeWrite + } else { + requestedMode = models.AccessModeRead + } + } + // Prohibit push to mirror repositories. if requestedMode > models.AccessModeRead && repo.IsMirror { fail("mirror repository is read-only", "") @@ -261,6 +281,40 @@ func runServ(c *cli.Context) error { } } + //LFS token authentication + + if verb == lfsAuthenticateVerb { + + url := fmt.Sprintf("%s%s/%s.git/info/lfs", setting.AppURL, repoUser.Name, repo.Name) + + now := time.Now().UTC() + token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{ + "repo": repo.ID, + "op": lfsVerb, + "exp": now.Add(5 * time.Minute).Unix(), + }) + + // Sign and get the complete encoded token as a string using the secret + tokenString, err := token.SignedString(setting.LFS.JWTSecretBytes) + if err != nil { + fail("Internal error", "Failed to sign JWT token: %v", err) + } + + tokenAuthentication := &models.LFSTokenResponse{ + Header: make(map[string]string), + Href: url, + } + tokenAuthentication.Header["Authorization"] = fmt.Sprintf("Bearer %s", tokenString) + + enc := json.NewEncoder(os.Stdout) + err = enc.Encode(tokenAuthentication) + if err != nil { + fail("Internal error", "Failed to encode LFS json response: %v", err) + } + + return nil + } + uuid := gouuid.NewV4().String() os.Setenv("GITEA_UUID", uuid) // Keep the old env variable name for backward compability diff --git a/models/lfs.go b/models/lfs.go index 24171fcf1436a..173917670add9 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -15,6 +15,11 @@ type LFSMetaObject struct { CreatedUnix int64 } +type LFSTokenResponse struct { + Header map[string]string `json:"header"` + Href string `json:"href"` +} + var ( ErrLFSObjectNotExist = errors.New("LFS Meta object does not exist") ) diff --git a/modules/lfs/server.go b/modules/lfs/server.go index 5b95ecb381f17..52ef8a8d91355 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -251,7 +251,12 @@ func (a *LFSHandler) BatchHandler(ctx *context.Context) { return } - if !authenticate(repository, object.Authorization, true) { + requireWrite := false + if bv.Operation == "upload" { + requireWrite = true + } + + if !authenticate(repository, object.Authorization, requireWrite) { requireAuth(ctx) return } @@ -428,9 +433,9 @@ func authenticate(repository *models.Repository, authorization string, requireWr return false } - //if authenticateToken(authorization, requireWrite) { - // return true - //} + if authenticateToken(repository, authorization, requireWrite) { + return true + } if !strings.HasPrefix(authorization, "Basic ") { return false @@ -465,7 +470,6 @@ func authenticate(repository *models.Repository, authorization string, requireWr return accessCheck } -/* func authenticateToken(repository *models.Repository, authorization string, requireWrite bool) bool { if !strings.HasPrefix(authorization, "Bearer ") { return false @@ -485,15 +489,26 @@ func authenticateToken(repository *models.Repository, authorization string, requ return false } - opstr, ok := claims["op"].(string) + opStr, ok := claims["op"].(string) + if !ok { + return false + } + + if requireWrite && opStr != "upload" { + return false + } + + repoId, ok := claims["repo"].(float64) if !ok { return false } - op := strings.ToLower(strings.TrimSpace(opstr)) - status := op == "upload" || (op == "download" && !requireWrite) - return status + + if repository.ID != int64(repoId) { + return false + } + + return true } -*/ type authError struct { error From 39a7b70b736d3d42e263d976fb1d3ca05fe26216 Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Mon, 7 Nov 2016 22:26:55 +0100 Subject: [PATCH 13/37] Integrate LFS settings into install process --- modules/auth/user_form.go | 1 + options/locale/locale_en-US.ini | 3 +++ routers/install.go | 20 ++++++++++++++++++-- templates/install.tmpl | 5 +++++ 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/modules/auth/user_form.go b/modules/auth/user_form.go index 0bdd7c1532531..f020f365d2a3d 100644 --- a/modules/auth/user_form.go +++ b/modules/auth/user_form.go @@ -23,6 +23,7 @@ type InstallForm struct { AppName string `binding:"Required" locale:"install.app_name"` RepoRootPath string `binding:"Required"` + LFSRootPath string RunUser string `binding:"Required"` Domain string `binding:"Required"` SSHPort int diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 97725a950ae14..566b5acfe7b9b 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -69,6 +69,8 @@ app_name = Application Name app_name_helper = Put your organization name here huge and loud! repo_path = Repository Root Path repo_path_helper = All Git remote repositories will be saved to this directory. +lfs_path = LFS Root Path +lfs_path_helper = Files stored with Git LFS will be stored in this directory. Leave empty to disable LFS. run_user = Run User run_user_helper = The user must have access to Repository Root Path and run Gitea. domain = Domain @@ -1059,6 +1061,7 @@ config.disable_router_log = Disable Router Log config.run_user = Run User config.run_mode = Run Mode config.repo_root_path = Repository Root Path +config.lfs_root_path = LFS Root Path config.static_file_root_path = Static File Root Path config.log_file_root_path = Log File Root Path config.script_type = Script Type diff --git a/routers/install.go b/routers/install.go index 78e9da0454d2a..08687612ddef0 100644 --- a/routers/install.go +++ b/routers/install.go @@ -79,6 +79,7 @@ func Install(ctx *context.Context) { // Application general settings form.AppName = setting.AppName form.RepoRootPath = setting.RepoRootPath + form.LFSRootPath = setting.LFS.ContentPath // Note(unknwon): it's hard for Windows users change a running user, // so just use current one if config says default. @@ -183,6 +184,16 @@ func InstallPost(ctx *context.Context, form auth.InstallForm) { return } + // Test LFS root path if not empty, empty meaning disable LFS + if form.LFSRootPath != "" { + form.LFSRootPath = strings.Replace(form.LFSRootPath, "\\", "/", -1) + if err := os.MkdirAll(form.LFSRootPath, os.ModePerm); err != nil { + ctx.Data["Err_LFSRootPath"] = true + ctx.RenderWithErr(ctx.Tr("install.invalid_lfs_path", err), tplInstall, &form) + return + } + } + // Test log root path. form.LogRootPath = strings.Replace(form.LogRootPath, "\\", "/", -1) if err = os.MkdirAll(form.LogRootPath, os.ModePerm); err != nil { @@ -254,8 +265,13 @@ func InstallPost(ctx *context.Context, form auth.InstallForm) { cfg.Section("server").Key("SSH_PORT").SetValue(com.ToStr(form.SSHPort)) } - //TODO Expose LFS settings in install form - cfg.Section("server").Key("LFS_JWT_SECRET").SetValue(base.GetRandomBytesAsBase64(32)) + if form.LFSRootPath != "" { + cfg.Section("server").Key("LFS_START_SERVER").SetValue("true") + cfg.Section("server").Key("LFS_CONTENT_PATH").SetValue(form.LFSRootPath) + cfg.Section("server").Key("LFS_JWT_SECRET").SetValue(base.GetRandomBytesAsBase64(32)) + } else { + cfg.Section("server").Key("LFS_START_SERVER").SetValue("false") + } if len(strings.TrimSpace(form.SMTPHost)) > 0 { cfg.Section("mailer").Key("ENABLED").SetValue("true") diff --git a/templates/install.tmpl b/templates/install.tmpl index 44851c96fd403..027d06166b43a 100644 --- a/templates/install.tmpl +++ b/templates/install.tmpl @@ -84,6 +84,11 @@ {{.i18n.Tr "install.repo_path_helper"}} +
+ + + {{.i18n.Tr "install.lfs_path_helper"}} +
From 2568f71522f7f976811db35b66800d6bc659439e Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 8 Nov 2016 15:38:11 +0100 Subject: [PATCH 14/37] Remove LFS objects when repository is deleted Only removes objects from content store when deleted repo is the only referencing repository --- models/lfs.go | 3 ++- models/repo.go | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/models/lfs.go b/models/lfs.go index 173917670add9..b2e63c27016cc 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -7,7 +7,8 @@ import ( ) type LFSMetaObject struct { - Oid string `xorm:"pk"` + ID int64 `xorm:"pk autoincr"` + Oid string `xorm:"INDEX NOT NULL"` Size int64 `xorm:"NOT NULL"` RepositoryID int64 `xorm:"NOT NULL"` Existing bool `xorm:"-"` diff --git a/models/repo.go b/models/repo.go index f6e1af55cdc19..61f556029f763 100644 --- a/models/repo.go +++ b/models/repo.go @@ -1480,6 +1480,36 @@ func DeleteRepository(uid, repoID int64) error { RemoveAllWithNotice("Delete attachment", attachmentPaths[i]) } + // Remove LFS objects + var lfsObjects []*LFSMetaObject + + if err = sess.Where("repository_id=?", repoID).Find(&lfsObjects); err != nil { + return err + } + + for _, v := range lfsObjects { + + //Check that no other repository references this object + var matchedOidObjects []*LFSMetaObject + if err = sess.Where("oid=?", v.Oid).Find(&matchedOidObjects); err != nil { + return err + } + + if len(matchedOidObjects) > 1 { + continue + } + + oidPath := filepath.Join(v.Oid[0:2], v.Oid[2:4], v.Oid[4:len(v.Oid)]) + err = os.Remove(filepath.Join(setting.LFS.ContentPath, oidPath)) + if err != nil { + return err + } + } + + if _, err = sess.Exec("DELETE FROM `lfs_meta_object` WHERE repository_id=?", repoID); err != nil { + return err + } + if err = sess.Commit(); err != nil { return fmt.Errorf("Commit: %v", err) } @@ -2240,6 +2270,34 @@ func ForkRepository(u *User, oldRepo *Repository, name, desc string) (_ *Reposit return nil, fmt.Errorf("createUpdateHook: %v", err) } + //Commit repo to get Fork ID + err = sess.Commit() + if err != nil { + return nil, err + } + sessionRelease(sess) + + // Copy LFS meta objects in new session + sess = x.NewSession() + defer sessionRelease(sess) + if err = sess.Begin(); err != nil { + return nil, err + } + + var lfsObjects []*LFSMetaObject + + if err = sess.Where("repository_id=?", oldRepo.ID).Find(&lfsObjects); err != nil { + return nil, err + } + + for _, v := range lfsObjects { + v.ID = 0 + v.RepositoryID = repo.ID + if _, err = sess.Insert(v); err != nil { + return nil, err + } + } + return repo, sess.Commit() } From 307d1fd09cbd2ec4913b1e4372ed3fd99eac336d Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 8 Nov 2016 17:13:52 +0100 Subject: [PATCH 15/37] Make LFS module stateless Fixes bug where LFS would not work after installation without restarting Gitea --- cmd/web.go | 15 +-- modules/lfs/content_store.go | 9 -- modules/lfs/content_store_test.go | 174 ------------------------------ modules/lfs/server.go | 66 ++++++------ 4 files changed, 40 insertions(+), 224 deletions(-) delete mode 100644 modules/lfs/content_store_test.go diff --git a/cmd/web.go b/cmd/web.go index 8785c6c4caa22..2b3f29adcea2f 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -565,16 +565,11 @@ func runWeb(ctx *cli.Context) error { }, ignSignIn, context.RepoAssignment(true), context.RepoRef()) m.Group("/:reponame", func() { - - if setting.LFS.StartServer { - lfsHandler := lfs.NewLFSHandler() - m.Group("/info/lfs", func() { - m.Post("/objects/batch", lfsHandler.BatchHandler) - m.Any("/objects/:oid", lfsHandler.ObjectOidHandler) - m.Post("/objects", lfsHandler.PostHandler) - }, ignSignInAndCsrf) - } - + m.Group("/info/lfs", func() { + m.Post("/objects/batch", lfs.BatchHandler) + m.Any("/objects/:oid", lfs.ObjectOidHandler) + m.Post("/objects", lfs.PostHandler) + }, ignSignInAndCsrf) m.Any("/*", ignSignInAndCsrf, repo.HTTP) m.Head("/tasks/trigger", repo.TriggerTask) }) diff --git a/modules/lfs/content_store.go b/modules/lfs/content_store.go index 3f31a0ede955e..c6228c926510e 100644 --- a/modules/lfs/content_store.go +++ b/modules/lfs/content_store.go @@ -20,15 +20,6 @@ type ContentStore struct { basePath string } -// NewContentStore creates a ContentStore at the base directory. -func NewContentStore(base string) (*ContentStore, error) { - if err := os.MkdirAll(base, 0750); err != nil { - return nil, err - } - - return &ContentStore{base}, nil -} - // Get takes a Meta object and retreives the content from the store, returning // it as an io.Reader. If fromByte > 0, the reader starts from that byte func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.Reader, error) { diff --git a/modules/lfs/content_store_test.go b/modules/lfs/content_store_test.go deleted file mode 100644 index e4bebfa55aa21..0000000000000 --- a/modules/lfs/content_store_test.go +++ /dev/null @@ -1,174 +0,0 @@ -package lfs - -import ( - "bytes" - "fmt" - "github.com/gogits/gogs/models" - "io/ioutil" - "os" - "testing" -) - -var contentStore *ContentStore - -func TestContenStorePut(t *testing.T) { - setup() - defer teardown() - - m := &models.LFSMetaObject{ - Oid: "6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72", - Size: 12, - } - - b := bytes.NewBuffer([]byte("test content")) - - if err := contentStore.Put(m, b); err != nil { - t.Fatalf("expected put to succeed, got: %s", err) - } - - path := "content-store-test/6a/e8/a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72" - if _, err := os.Stat(path); os.IsNotExist(err) { - t.Fatalf("expected content to exist after putting") - } -} - -func TestContenStorePutHashMismatch(t *testing.T) { - setup() - defer teardown() - - m := &models.LFSMetaObject{ - Oid: "6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72", - Size: 12, - } - - b := bytes.NewBuffer([]byte("bogus content")) - - if err := contentStore.Put(m, b); err == nil { - t.Fatal("expected put with bogus content to fail") - } - - path := "content-store-test/6a/e8/a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72" - if _, err := os.Stat(path); err == nil { - t.Fatalf("expected content to not exist after putting bogus content") - } -} - -func TestContenStorePutSizeMismatch(t *testing.T) { - setup() - defer teardown() - - m := &models.LFSMetaObject{ - Oid: "6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72", - Size: 14, - } - - b := bytes.NewBuffer([]byte("test content")) - - if err := contentStore.Put(m, b); err == nil { - t.Fatal("expected put with bogus size to fail") - } - - path := "content-store-test/6a/e8/a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72" - if _, err := os.Stat(path); err == nil { - t.Fatalf("expected content to not exist after putting bogus size") - } -} - -func TestContenStoreGet(t *testing.T) { - setup() - defer teardown() - - m := &models.LFSMetaObject{ - Oid: "6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72", - Size: 12, - } - - b := bytes.NewBuffer([]byte("test content")) - - if err := contentStore.Put(m, b); err != nil { - t.Fatalf("expected put to succeed, got: %s", err) - } - - r, err := contentStore.Get(m, 0) - if err != nil { - t.Fatalf("expected get to succeed, got: %s", err) - } - - by, _ := ioutil.ReadAll(r) - if string(by) != "test content" { - t.Fatalf("expected to read content, got: %s", string(by)) - } -} - -func TestContenStoreGetWithRange(t *testing.T) { - setup() - defer teardown() - - m := &models.LFSMetaObject{ - Oid: "6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72", - Size: 12, - } - - b := bytes.NewBuffer([]byte("test content")) - - if err := contentStore.Put(m, b); err != nil { - t.Fatalf("expected put to succeed, got: %s", err) - } - - r, err := contentStore.Get(m, 5) - if err != nil { - t.Fatalf("expected get to succeed, got: %s", err) - } - - by, _ := ioutil.ReadAll(r) - if string(by) != "content" { - t.Fatalf("expected to read content, got: %s", string(by)) - } -} - -func TestContenStoreGetNonExisting(t *testing.T) { - setup() - defer teardown() - - _, err := contentStore.Get(&models.LFSMetaObject{Oid: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, 0) - if err == nil { - t.Fatalf("expected to get an error, but content existed") - } -} - -func TestContenStoreExists(t *testing.T) { - setup() - defer teardown() - - m := &models.LFSMetaObject{ - Oid: "6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72", - Size: 12, - } - - b := bytes.NewBuffer([]byte("test content")) - - if contentStore.Exists(m) { - t.Fatalf("expected content to not exist yet") - } - - if err := contentStore.Put(m, b); err != nil { - t.Fatalf("expected put to succeed, got: %s", err) - } - - if !contentStore.Exists(m) { - t.Fatalf("expected content to exist") - } -} - -func setup() { - store, err := NewContentStore("content-store-test") - if err != nil { - fmt.Printf("error initializing content store: %s\n", err) - os.Exit(1) - } - contentStore = store -} - -func teardown() { - os.RemoveAll("content-store-test") -} diff --git a/modules/lfs/server.go b/modules/lfs/server.go index 52ef8a8d91355..00e58d8eec00a 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -71,41 +71,31 @@ type link struct { ExpiresAt time.Time `json:"expires_at,omitempty"` } -type LFSHandler struct { - contentStore *ContentStore -} - -func NewLFSHandler() *LFSHandler { - contentStore, err := NewContentStore(setting.LFS.ContentPath) +func ObjectOidHandler(ctx *context.Context) { - if err != nil { - log.Fatal(4, "Error initializing LFS content store: %s", err) + if !setting.LFS.StartServer { + writeStatus(ctx, 404) + return } - app := &LFSHandler{contentStore: contentStore} - return app -} - -func (a *LFSHandler) ObjectOidHandler(ctx *context.Context) { - if ctx.Req.Method == "GET" || ctx.Req.Method == "HEAD" { if MetaMatcher(ctx.Req) { - a.GetMetaHandler(ctx) + GetMetaHandler(ctx) return } if ContentMatcher(ctx.Req) { - a.GetContentHandler(ctx) + GetContentHandler(ctx) return } } else if ctx.Req.Method == "PUT" && ContentMatcher(ctx.Req) { - a.PutHandler(ctx) + PutHandler(ctx) return } } // GetContentHandler gets the content from the content store -func (a *LFSHandler) GetContentHandler(ctx *context.Context) { +func GetContentHandler(ctx *context.Context) { rv := unpack(ctx) @@ -140,7 +130,8 @@ func (a *LFSHandler) GetContentHandler(ctx *context.Context) { } } - content, err := a.contentStore.Get(meta, fromByte) + contentStore := &ContentStore{basePath: setting.LFS.ContentPath} + content, err := contentStore.Get(meta, fromByte) if err != nil { writeStatus(ctx, 404) return @@ -152,7 +143,7 @@ func (a *LFSHandler) GetContentHandler(ctx *context.Context) { } // GetMetaHandler retrieves metadata about the object -func (a *LFSHandler) GetMetaHandler(ctx *context.Context) { +func GetMetaHandler(ctx *context.Context) { rv := unpack(ctx) @@ -178,14 +169,19 @@ func (a *LFSHandler) GetMetaHandler(ctx *context.Context) { if ctx.Req.Method == "GET" { enc := json.NewEncoder(ctx.Resp) - enc.Encode(a.Represent(rv, meta, true, false)) + enc.Encode(Represent(rv, meta, true, false)) } logRequest(ctx.Req, 200) } // PostHandler instructs the client how to upload data -func (a *LFSHandler) PostHandler(ctx *context.Context) { +func PostHandler(ctx *context.Context) { + + if !setting.LFS.StartServer { + writeStatus(ctx, 404) + return + } if !MetaMatcher(ctx.Req) { writeStatus(ctx, 500) @@ -217,18 +213,24 @@ func (a *LFSHandler) PostHandler(ctx *context.Context) { ctx.Resp.Header().Set("Content-Type", metaMediaType) sentStatus := 202 - if meta.Existing && a.contentStore.Exists(meta) { + contentStore := &ContentStore{basePath: setting.LFS.ContentPath} + if meta.Existing && contentStore.Exists(meta) { sentStatus = 200 } ctx.Resp.WriteHeader(sentStatus) enc := json.NewEncoder(ctx.Resp) - enc.Encode(a.Represent(rv, meta, meta.Existing, true)) + enc.Encode(Represent(rv, meta, meta.Existing, true)) logRequest(ctx.Req, sentStatus) } // BatchHandler provides the batch api -func (a *LFSHandler) BatchHandler(ctx *context.Context) { +func BatchHandler(ctx *context.Context) { + + if !setting.LFS.StartServer { + writeStatus(ctx, 404) + return + } if !MetaMatcher(ctx.Req) { writeStatus(ctx, 500) @@ -263,8 +265,9 @@ func (a *LFSHandler) BatchHandler(ctx *context.Context) { meta, err := models.GetLFSMetaObjectByOid(object.Oid) - if err == nil && a.contentStore.Exists(meta) { // Object is found and exists - responseObjects = append(responseObjects, a.Represent(object, meta, true, false)) + contentStore := &ContentStore{basePath: setting.LFS.ContentPath} + if err == nil && contentStore.Exists(meta) { // Object is found and exists + responseObjects = append(responseObjects, Represent(object, meta, true, false)) continue } @@ -272,7 +275,7 @@ func (a *LFSHandler) BatchHandler(ctx *context.Context) { meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: object.Oid, Size: object.Size, RepositoryID: repository.ID}) if err == nil { - responseObjects = append(responseObjects, a.Represent(object, meta, meta.Existing, true)) + responseObjects = append(responseObjects, Represent(object, meta, meta.Existing, true)) } } @@ -286,7 +289,7 @@ func (a *LFSHandler) BatchHandler(ctx *context.Context) { } // PutHandler receives data from the client and puts it into the content store -func (a *LFSHandler) PutHandler(ctx *context.Context) { +func PutHandler(ctx *context.Context) { rv := unpack(ctx) meta, err := models.GetLFSMetaObjectByOid(rv.Oid) @@ -308,7 +311,8 @@ func (a *LFSHandler) PutHandler(ctx *context.Context) { return } - if err := a.contentStore.Put(meta, ctx.Req.Body().ReadCloser()); err != nil { + contentStore := &ContentStore{basePath: setting.LFS.ContentPath} + if err := contentStore.Put(meta, ctx.Req.Body().ReadCloser()); err != nil { models.RemoveLFSMetaObjectByOid(rv.Oid) ctx.Resp.WriteHeader(500) fmt.Fprintf(ctx.Resp, `{"message":"%s"}`, err) @@ -320,7 +324,7 @@ func (a *LFSHandler) PutHandler(ctx *context.Context) { // Represent takes a RequestVars and Meta and turns it into a Representation suitable // for json encoding -func (a *LFSHandler) Represent(rv *RequestVars, meta *models.LFSMetaObject, download, upload bool) *Representation { +func Represent(rv *RequestVars, meta *models.LFSMetaObject, download, upload bool) *Representation { rep := &Representation{ Oid: meta.Oid, Size: meta.Size, From f5369783b1ed4e370888461f0e54aee39813884a Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 9 Nov 2016 12:42:23 +0100 Subject: [PATCH 16/37] Change 500 'Internal Server Error' to 400 'Bad Request' --- modules/lfs/server.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/lfs/server.go b/modules/lfs/server.go index 00e58d8eec00a..67d8e99770eb4 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -184,7 +184,7 @@ func PostHandler(ctx *context.Context) { } if !MetaMatcher(ctx.Req) { - writeStatus(ctx, 500) + writeStatus(ctx, 400) return } @@ -233,7 +233,7 @@ func BatchHandler(ctx *context.Context) { } if !MetaMatcher(ctx.Req) { - writeStatus(ctx, 500) + writeStatus(ctx, 400) return } From f528c0221e1679e717f9af077622192f78164c8b Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Sat, 12 Nov 2016 02:02:11 +0100 Subject: [PATCH 17/37] Change sql query to xorm call --- models/repo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/repo.go b/models/repo.go index 61f556029f763..2cdfef8932bd4 100644 --- a/models/repo.go +++ b/models/repo.go @@ -1506,7 +1506,7 @@ func DeleteRepository(uid, repoID int64) error { } } - if _, err = sess.Exec("DELETE FROM `lfs_meta_object` WHERE repository_id=?", repoID); err != nil { + if _, err := sess.Delete(&LFSMetaObject{RepositoryID: repoID}); err != nil { return err } From d769db21e6017208eeeeb00c52c3b04aa5f99083 Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Sat, 12 Nov 2016 02:03:44 +0100 Subject: [PATCH 18/37] Remove unneeded type from LFS module --- modules/lfs/server.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/modules/lfs/server.go b/modules/lfs/server.go index 67d8e99770eb4..46854bdf430d9 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -514,14 +514,6 @@ func authenticateToken(repository *models.Repository, authorization string, requ return true } -type authError struct { - error -} - -func (e authError) AuthError() bool { - return true -} - func requireAuth(ctx *context.Context) { ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gogs-lfs") writeStatus(ctx, 401) From 18a28f529f273ffcc18df79f0db875ca6cf09c7e Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Sat, 12 Nov 2016 02:28:43 +0100 Subject: [PATCH 19/37] Change internal imports to code.gitea.io/gitea/ --- modules/lfs/content_store.go | 2 +- modules/lfs/server.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/lfs/content_store.go b/modules/lfs/content_store.go index c6228c926510e..d45a2f447f077 100644 --- a/modules/lfs/content_store.go +++ b/modules/lfs/content_store.go @@ -4,7 +4,7 @@ import ( "crypto/sha256" "encoding/hex" "errors" - "github.com/go-gitea/gitea/models" + "code.gitea.io/gitea/models" "io" "os" "path/filepath" diff --git a/modules/lfs/server.go b/modules/lfs/server.go index 46854bdf430d9..7937e755df84d 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -12,10 +12,10 @@ import ( "time" "github.com/dgrijalva/jwt-go" - "github.com/go-gitea/gitea/models" - "github.com/go-gitea/gitea/modules/context" - "github.com/go-gitea/gitea/modules/log" - "github.com/go-gitea/gitea/modules/setting" + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" "gopkg.in/macaron.v1" ) From 5d5962b77e3738b8192ef4a2df5bfb0ba0856ee8 Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Sat, 12 Nov 2016 16:16:04 +0100 Subject: [PATCH 20/37] Add Gitea authors copyright --- modules/lfs/LICENSE | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/lfs/LICENSE b/modules/lfs/LICENSE index 3d7c5a5c51101..0a94a80c8c2a3 100644 --- a/modules/lfs/LICENSE +++ b/modules/lfs/LICENSE @@ -1,3 +1,4 @@ +Copyright (c) 2016 The Gitea Authors Copyright (c) GitHub, Inc. and LFS Test Server contributors Permission is hereby granted, free of charge, to any person obtaining a copy From e2fe2e34b9f8a92b355888f01f38b792cb439954 Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Sun, 13 Nov 2016 12:38:42 +0100 Subject: [PATCH 21/37] Change basic auth realm to "gitea-lfs" --- modules/lfs/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/lfs/server.go b/modules/lfs/server.go index 7937e755df84d..1a943ec4fda32 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -515,6 +515,6 @@ func authenticateToken(repository *models.Repository, authorization string, requ } func requireAuth(ctx *context.Context) { - ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gogs-lfs") + ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs") writeStatus(ctx, 401) } From d27ee44768b2c4d73bb402e9830433af9135b6e8 Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Sun, 13 Nov 2016 12:42:28 +0100 Subject: [PATCH 22/37] Add unique indexes to LFS model --- models/lfs.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/lfs.go b/models/lfs.go index b2e63c27016cc..7eaf6cc007ade 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -8,9 +8,9 @@ import ( type LFSMetaObject struct { ID int64 `xorm:"pk autoincr"` - Oid string `xorm:"INDEX NOT NULL"` + Oid string `xorm:"UNIQUE(s) INDEX NOT NULL"` Size int64 `xorm:"NOT NULL"` - RepositoryID int64 `xorm:"NOT NULL"` + RepositoryID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` Existing bool `xorm:"-"` Created time.Time `xorm:"-"` CreatedUnix int64 From c0245731ea6a8ad73267d67a6cd2df08d7b9ef20 Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Sun, 13 Nov 2016 12:46:56 +0100 Subject: [PATCH 23/37] Use xorm count function in LFS check on repository delete --- models/repo.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/models/repo.go b/models/repo.go index 2cdfef8932bd4..64369dabbbbb0 100644 --- a/models/repo.go +++ b/models/repo.go @@ -1488,14 +1488,13 @@ func DeleteRepository(uid, repoID int64) error { } for _, v := range lfsObjects { + count, err := sess.Count(&LFSMetaObject{Oid: v.Oid}) - //Check that no other repository references this object - var matchedOidObjects []*LFSMetaObject - if err = sess.Where("oid=?", v.Oid).Find(&matchedOidObjects); err != nil { + if err != nil { return err } - if len(matchedOidObjects) > 1 { + if count > 1 { continue } From 4683c7dedc06e91f011a6f835e86da2b6e326fb7 Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Sun, 13 Nov 2016 12:50:54 +0100 Subject: [PATCH 24/37] Return io.ReadCloser from content store and close after usage --- modules/lfs/content_store.go | 4 ++-- modules/lfs/server.go | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/lfs/content_store.go b/modules/lfs/content_store.go index d45a2f447f077..40490528c298b 100644 --- a/modules/lfs/content_store.go +++ b/modules/lfs/content_store.go @@ -1,10 +1,10 @@ package lfs import ( + "code.gitea.io/gitea/models" "crypto/sha256" "encoding/hex" "errors" - "code.gitea.io/gitea/models" "io" "os" "path/filepath" @@ -22,7 +22,7 @@ type ContentStore struct { // Get takes a Meta object and retreives the content from the store, returning // it as an io.Reader. If fromByte > 0, the reader starts from that byte -func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.Reader, error) { +func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadCloser, error) { path := filepath.Join(s.basePath, transformKey(meta.Oid)) f, err := os.Open(path) diff --git a/modules/lfs/server.go b/modules/lfs/server.go index 1a943ec4fda32..d4c60bf72bac6 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -11,11 +11,11 @@ import ( "strings" "time" - "github.com/dgrijalva/jwt-go" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "github.com/dgrijalva/jwt-go" "gopkg.in/macaron.v1" ) @@ -139,6 +139,7 @@ func GetContentHandler(ctx *context.Context) { ctx.Resp.WriteHeader(statusCode) io.Copy(ctx.Resp, content) + content.Close() logRequest(ctx.Req, statusCode) } From 71dae631a4b12a9ca34251c2b1d8c01f0d0ed58c Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Sun, 13 Nov 2016 14:44:28 +0100 Subject: [PATCH 25/37] Add LFS info to runWeb() --- cmd/web.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/web.go b/cmd/web.go index 2b3f29adcea2f..ad9c97dda1f8a 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -606,6 +606,10 @@ func runWeb(ctx *cli.Context) error { } log.Info("Listen: %v://%s%s", setting.Protocol, listenAddr, setting.AppSubURL) + if setting.LFS.StartServer { + log.Info("LFS server enabled") + } + var err error switch setting.Protocol { case setting.HTTP: From 0daa396049e68c76fc153253cfd94f79bbed609f Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 14 Nov 2016 16:14:04 +0100 Subject: [PATCH 26/37] Export LFS content store base path --- modules/lfs/content_store.go | 8 ++++---- modules/lfs/server.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/lfs/content_store.go b/modules/lfs/content_store.go index 40490528c298b..2ca44512f1989 100644 --- a/modules/lfs/content_store.go +++ b/modules/lfs/content_store.go @@ -17,13 +17,13 @@ var ( // ContentStore provides a simple file system based storage. type ContentStore struct { - basePath string + BasePath string } // Get takes a Meta object and retreives the content from the store, returning // it as an io.Reader. If fromByte > 0, the reader starts from that byte func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadCloser, error) { - path := filepath.Join(s.basePath, transformKey(meta.Oid)) + path := filepath.Join(s.BasePath, transformKey(meta.Oid)) f, err := os.Open(path) if err != nil { @@ -37,7 +37,7 @@ func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadC // Put takes a Meta object and an io.Reader and writes the content to the store. func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error { - path := filepath.Join(s.basePath, transformKey(meta.Oid)) + path := filepath.Join(s.BasePath, transformKey(meta.Oid)) tmpPath := path + ".tmp" dir := filepath.Dir(path) @@ -78,7 +78,7 @@ func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error { // Exists returns true if the object exists in the content store. func (s *ContentStore) Exists(meta *models.LFSMetaObject) bool { - path := filepath.Join(s.basePath, transformKey(meta.Oid)) + path := filepath.Join(s.BasePath, transformKey(meta.Oid)) if _, err := os.Stat(path); os.IsNotExist(err) { return false } diff --git a/modules/lfs/server.go b/modules/lfs/server.go index d4c60bf72bac6..346671e1b9649 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -130,7 +130,7 @@ func GetContentHandler(ctx *context.Context) { } } - contentStore := &ContentStore{basePath: setting.LFS.ContentPath} + contentStore := &ContentStore{BasePath: setting.LFS.ContentPath} content, err := contentStore.Get(meta, fromByte) if err != nil { writeStatus(ctx, 404) @@ -214,7 +214,7 @@ func PostHandler(ctx *context.Context) { ctx.Resp.Header().Set("Content-Type", metaMediaType) sentStatus := 202 - contentStore := &ContentStore{basePath: setting.LFS.ContentPath} + contentStore := &ContentStore{BasePath: setting.LFS.ContentPath} if meta.Existing && contentStore.Exists(meta) { sentStatus = 200 } @@ -266,7 +266,7 @@ func BatchHandler(ctx *context.Context) { meta, err := models.GetLFSMetaObjectByOid(object.Oid) - contentStore := &ContentStore{basePath: setting.LFS.ContentPath} + contentStore := &ContentStore{BasePath: setting.LFS.ContentPath} if err == nil && contentStore.Exists(meta) { // Object is found and exists responseObjects = append(responseObjects, Represent(object, meta, true, false)) continue @@ -312,7 +312,7 @@ func PutHandler(ctx *context.Context) { return } - contentStore := &ContentStore{basePath: setting.LFS.ContentPath} + contentStore := &ContentStore{BasePath: setting.LFS.ContentPath} if err := contentStore.Put(meta, ctx.Req.Body().ReadCloser()); err != nil { models.RemoveLFSMetaObjectByOid(rv.Oid) ctx.Resp.WriteHeader(500) From 9f10ad9ad850a28127a1a868938295e812c7e114 Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Mon, 14 Nov 2016 20:31:02 +0100 Subject: [PATCH 27/37] LFS file download from UI --- cmd/web.go | 1 + modules/lfs/server.go | 40 +++++++++++++++++++++++---------- options/locale/locale_en-US.ini | 1 + routers/repo/view.go | 26 +++++++++++++++++++++ templates/repo/view_file.tmpl | 4 ++-- 5 files changed, 58 insertions(+), 14 deletions(-) diff --git a/cmd/web.go b/cmd/web.go index ad9c97dda1f8a..fe2575ecc7bd5 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -567,6 +567,7 @@ func runWeb(ctx *cli.Context) error { m.Group("/:reponame", func() { m.Group("/info/lfs", func() { m.Post("/objects/batch", lfs.BatchHandler) + m.Get("/objects/:oid/:filename", lfs.ObjectOidHandler) m.Any("/objects/:oid", lfs.ObjectOidHandler) m.Post("/objects", lfs.PostHandler) }, ignSignInAndCsrf) diff --git a/modules/lfs/server.go b/modules/lfs/server.go index 346671e1b9649..85b129577b59b 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -83,7 +83,7 @@ func ObjectOidHandler(ctx *context.Context) { GetMetaHandler(ctx) return } - if ContentMatcher(ctx.Req) { + if ContentMatcher(ctx.Req) || ctx.IsSigned { GetContentHandler(ctx) return } @@ -112,7 +112,7 @@ func GetContentHandler(ctx *context.Context) { return } - if !authenticate(repository, rv.Authorization, false) { + if !authenticate(ctx, repository, rv.Authorization, false) { requireAuth(ctx) return } @@ -137,6 +137,17 @@ func GetContentHandler(ctx *context.Context) { return } + ctx.Resp.Header().Set("Content-Length", strconv.FormatInt(meta.Size, 10)) + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + + filename := ctx.Params("filename") + if len(filename) > 0 { + decodedFilename, err := base64.RawURLEncoding.DecodeString(filename) + if err == nil { + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename=\""+string(decodedFilename)+"\"") + } + } + ctx.Resp.WriteHeader(statusCode) io.Copy(ctx.Resp, content) content.Close() @@ -161,7 +172,7 @@ func GetMetaHandler(ctx *context.Context) { return } - if !authenticate(repository, rv.Authorization, false) { + if !authenticate(ctx, repository, rv.Authorization, false) { requireAuth(ctx) return } @@ -200,7 +211,7 @@ func PostHandler(ctx *context.Context) { return } - if !authenticate(repository, rv.Authorization, true) { + if !authenticate(ctx, repository, rv.Authorization, true) { requireAuth(ctx) } @@ -259,7 +270,7 @@ func BatchHandler(ctx *context.Context) { requireWrite = true } - if !authenticate(repository, object.Authorization, requireWrite) { + if !authenticate(ctx, repository, object.Authorization, requireWrite) { requireAuth(ctx) return } @@ -307,7 +318,7 @@ func PutHandler(ctx *context.Context) { return } - if !authenticate(repository, rv.Authorization, true) { + if !authenticate(ctx, repository, rv.Authorization, true) { requireAuth(ctx) return } @@ -428,7 +439,17 @@ func logRequest(r macaron.Request, status int) { // authenticate uses the authorization string to determine whether // or not to proceed. This server assumes an HTTP Basic auth format. -func authenticate(repository *models.Repository, authorization string, requireWrite bool) bool { +func authenticate(ctx *context.Context, repository *models.Repository, authorization string, requireWrite bool) bool { + + accessMode := models.AccessModeRead + if requireWrite { + accessMode = models.AccessModeWrite + } + + if ctx.IsSigned { + accessCheck, _ := models.HasAccess(ctx.User, repository, accessMode) + return accessCheck + } if !repository.IsPrivate && !requireWrite { return true @@ -466,11 +487,6 @@ func authenticate(repository *models.Repository, authorization string, requireWr return false } - accessMode := models.AccessModeRead - if requireWrite { - accessMode = models.AccessModeWrite - } - accessCheck, _ := models.HasAccess(userModel, repository, accessMode) return accessCheck } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 566b5acfe7b9b..105e76eb749c9 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -434,6 +434,7 @@ file_view_raw = View Raw file_permalink = Permalink file_too_large = This file is too large to be shown video_not_supported_in_browser = Your browser doesn't support HTML5 video tag. +stored_lfs = Stored with Git LFS editor.new_file = New file editor.upload_file = Upload file diff --git a/routers/repo/view.go b/routers/repo/view.go index 7e7ad0b923ee8..c2bb7cab2aec5 100644 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -17,11 +17,14 @@ import ( "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/highlight" + "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markdown" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/templates" + "encoding/base64" "github.com/Unknwon/paginater" + "strconv" ) const ( @@ -139,6 +142,29 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st isTextFile := base.IsTextFile(buf) ctx.Data["IsTextFile"] = isTextFile + //Check for LFS meta file + if isTextFile { + headString := string(buf) + if strings.HasPrefix(headString, "version https://git-lfs.github.com/spec/v1") { + splitLines := strings.Split(headString, "\n") + if len(splitLines) >= 3 { + oid := strings.TrimPrefix(splitLines[1], "oid sha256:") + size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64) + if len(oid) == 64 && err == nil { + contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath} + meta := &models.LFSMetaObject{Oid: oid} + if contentStore.Exists(meta) { + ctx.Data["IsTextFile"] = false + ctx.Data["IsLFSFile"] = true + ctx.Data["FileSize"] = size + filenameBase64 := base64.RawURLEncoding.EncodeToString([]byte(blob.Name())) + ctx.Data["RawFileLink"] = fmt.Sprintf("%s%s/info/lfs/objects/%s/%s", setting.AppURL, ctx.Repo.Repository.FullName(), oid, filenameBase64) + } + } + } + } + } + // Assume file is not editable first. if !isTextFile { ctx.Data["EditFileTooltip"] = ctx.Tr("repo.editor.cannot_edit_non_text_files") diff --git a/templates/repo/view_file.tmpl b/templates/repo/view_file.tmpl index 08d77c3f37e6e..9382bb075e0b7 100644 --- a/templates/repo/view_file.tmpl +++ b/templates/repo/view_file.tmpl @@ -5,11 +5,11 @@ {{if .ReadmeInList}} {{.FileName}} {{else}} - {{.FileName}} {{FileSize .FileSize}} + {{.FileName}} {{FileSize .FileSize}}{{if .IsLFSFile}} ({{.i18n.Tr "repo.stored_lfs"}}){{end}} {{end}} {{else}} - {{.FileName}} {{FileSize .FileSize}} + {{.FileName}} {{FileSize .FileSize}}{{if .IsLFSFile}} ({{.i18n.Tr "repo.stored_lfs"}}){{end}} {{end}} {{if not .ReadmeInList}}
From 5c18b29cb96a7610ad6deb480e5e55f5af12552f Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Mon, 14 Nov 2016 21:33:28 +0100 Subject: [PATCH 28/37] Work around git-lfs client issue with unauthenticated requests Returning a dummy Authorization header for unauthenticated requests lets git-lfs client skip asking for auth credentials See: https://github.com/github/git-lfs/issues/1088 --- modules/lfs/server.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/modules/lfs/server.go b/modules/lfs/server.go index 85b129577b59b..8f08dc0cac338 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -345,7 +345,14 @@ func Represent(rv *RequestVars, meta *models.LFSMetaObject, download, upload boo header := make(map[string]string) header["Accept"] = contentMediaType - header["Authorization"] = rv.Authorization + + if rv.Authorization == "" { + //https://github.com/github/git-lfs/issues/1088 + header["Authorization"] = "Authorization: Basic dummy" + } else { + header["Authorization"] = rv.Authorization + } + if download { rep.Actions["download"] = &link{Href: rv.ObjectLink(), Header: header} From 47691fa1843630d947dd5b77d8594b0de5106863 Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Mon, 14 Nov 2016 22:17:58 +0100 Subject: [PATCH 29/37] Fix unauthenticated UI downloads from public repositories --- modules/lfs/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/lfs/server.go b/modules/lfs/server.go index 8f08dc0cac338..c164bfb87e5a8 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -83,7 +83,7 @@ func ObjectOidHandler(ctx *context.Context) { GetMetaHandler(ctx) return } - if ContentMatcher(ctx.Req) || ctx.IsSigned { + if ContentMatcher(ctx.Req) || len(ctx.Params("filename")) > 0 { GetContentHandler(ctx) return } From cb0f5156b12824fce120b2fe7d66012404ee88ef Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Tue, 15 Nov 2016 00:59:41 +0100 Subject: [PATCH 30/37] Authentication check order, Finish LFS file view logic --- modules/lfs/server.go | 9 ++++----- routers/repo/view.go | 3 ++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/lfs/server.go b/modules/lfs/server.go index c164bfb87e5a8..804e8330fceb8 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -353,7 +353,6 @@ func Represent(rv *RequestVars, meta *models.LFSMetaObject, download, upload boo header["Authorization"] = rv.Authorization } - if download { rep.Actions["download"] = &link{Href: rv.ObjectLink(), Header: header} } @@ -453,15 +452,15 @@ func authenticate(ctx *context.Context, repository *models.Repository, authoriza accessMode = models.AccessModeWrite } + if !repository.IsPrivate && !requireWrite { + return true + } + if ctx.IsSigned { accessCheck, _ := models.HasAccess(ctx.User, repository, accessMode) return accessCheck } - if !repository.IsPrivate && !requireWrite { - return true - } - if authorization == "" { return false } diff --git a/routers/repo/view.go b/routers/repo/view.go index c2bb7cab2aec5..2f3bd03d4508c 100644 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -143,7 +143,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st ctx.Data["IsTextFile"] = isTextFile //Check for LFS meta file - if isTextFile { + if isTextFile && setting.LFS.StartServer { headString := string(buf) if strings.HasPrefix(headString, "version https://git-lfs.github.com/spec/v1") { splitLines := strings.Split(headString, "\n") @@ -155,6 +155,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st meta := &models.LFSMetaObject{Oid: oid} if contentStore.Exists(meta) { ctx.Data["IsTextFile"] = false + isTextFile = false ctx.Data["IsLFSFile"] = true ctx.Data["FileSize"] = size filenameBase64 := base64.RawURLEncoding.EncodeToString([]byte(blob.Name())) From 6deed2d1a6c143123ce9694ae3b752c91ede48d5 Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Thu, 17 Nov 2016 21:17:36 +0100 Subject: [PATCH 31/37] Ignore LFS hooks if installed for current OS user Fixes Gitea UI actions for repositories tracking LFS files. Checks for minimum needed git version by parsing the semantic version string. --- modules/setting/setting.go | 77 +++++++++++++++++++++++++++----------- 1 file changed, 56 insertions(+), 21 deletions(-) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 66750b28e81aa..125fa1cde2b37 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -20,6 +20,7 @@ import ( "strings" "time" + "code.gitea.io/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/user" "github.com/Unknwon/com" @@ -602,35 +603,69 @@ please consider changing to GITEA_CUSTOM`) log.Fatal(4, "Fail to create '%s': %v", LFS.ContentPath, err) } - LFS.JWTSecretBytes = make([]byte, 32) - n, err := base64.RawURLEncoding.Decode(LFS.JWTSecretBytes, []byte(LFS.JWTSecretBase64)) + //Disable LFS client hooks if installed for the current OS user + //Needs at least git v2.1.2 - if err != nil || n != 32 { - //Generate new secret and save to config + binVersion, err := git.BinVersion() + if err != nil { + log.Fatal(4, "Error retrieving git version: %s", err) + } - _, err := io.ReadFull(rand.Reader, LFS.JWTSecretBytes) + splitVersion := strings.SplitN(binVersion, ".", 3) - if err != nil { - log.Fatal(4, "Error reading random bytes: %s", err) - } + majorVersion, err := strconv.ParseUint(splitVersion[0], 10, 64) + if err != nil { + log.Fatal(4, "Error parsing git major version: %s", err) + } + minorVersion, err := strconv.ParseUint(splitVersion[1], 10, 64) + if err != nil { + log.Fatal(4, "Error parsing git minor version: %s", err) + } + revisionVersion, err := strconv.ParseUint(splitVersion[2], 10, 64) + if err != nil { + log.Fatal(4, "Error parsing git revision version: %s", err) + } - LFS.JWTSecretBase64 = base64.RawURLEncoding.EncodeToString(LFS.JWTSecretBytes) + if !((majorVersion > 2) || (majorVersion == 2 && minorVersion > 1) || + (majorVersion == 2 && minorVersion == 1 && revisionVersion >= 2)) { - // Save secret - cfg := ini.Empty() - if com.IsFile(CustomConf) { - // Keeps custom settings if there is already something. - if err := cfg.Append(CustomConf); err != nil { - log.Error(4, "Fail to load custom conf '%s': %v", CustomConf, err) + log.Error(4, "LFS server support needs at least Git v2.1.2") + + } else { + + git.GlobalCommandArgs = append(git.GlobalCommandArgs, "-c", "filter.lfs.required=", + "-c", "filter.lfs.smudge=", "-c", "filter.lfs.clean=") + + LFS.JWTSecretBytes = make([]byte, 32) + n, err := base64.RawURLEncoding.Decode(LFS.JWTSecretBytes, []byte(LFS.JWTSecretBase64)) + + if err != nil || n != 32 { + //Generate new secret and save to config + + _, err := io.ReadFull(rand.Reader, LFS.JWTSecretBytes) + + if err != nil { + log.Fatal(4, "Error reading random bytes: %s", err) } - } - cfg.Section("server").Key("LFS_JWT_SECRET").SetValue(LFS.JWTSecretBase64) + LFS.JWTSecretBase64 = base64.RawURLEncoding.EncodeToString(LFS.JWTSecretBytes) + + // Save secret + cfg := ini.Empty() + if com.IsFile(CustomConf) { + // Keeps custom settings if there is already something. + if err := cfg.Append(CustomConf); err != nil { + log.Error(4, "Fail to load custom conf '%s': %v", CustomConf, err) + } + } - os.MkdirAll(filepath.Dir(CustomConf), os.ModePerm) - if err := cfg.SaveTo(CustomConf); err != nil { - log.Fatal(4, "Error saving generated JWT Secret to custom config: %v", err) - return + cfg.Section("server").Key("LFS_JWT_SECRET").SetValue(LFS.JWTSecretBase64) + + os.MkdirAll(filepath.Dir(CustomConf), os.ModePerm) + if err := cfg.SaveTo(CustomConf); err != nil { + log.Fatal(4, "Error saving generated JWT Secret to custom config: %v", err) + return + } } } } From 728f52a2ca829049664eb58c51b547058f19dc7b Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Sun, 20 Nov 2016 14:18:54 +0100 Subject: [PATCH 32/37] Hide LFS metafile diff from commit view, marking as binary --- models/git_diff.go | 23 +++++++++++++++++++++++ models/lfs.go | 5 +++++ routers/repo/view.go | 4 ++-- 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/models/git_diff.go b/models/git_diff.go index fe99f61bfea18..e8b97a04e7a63 100644 --- a/models/git_diff.go +++ b/models/git_diff.go @@ -245,6 +245,7 @@ func ParsePatch(maxLines, maxLineCharacteres, maxFiles int, reader io.Reader) (* leftLine, rightLine int lineCount int curFileLinesCount int + curFileLFSPrefix bool ) input := bufio.NewReader(reader) @@ -268,6 +269,27 @@ func ParsePatch(maxLines, maxLineCharacteres, maxFiles int, reader io.Reader) (* continue } + trimLine := strings.Trim(line, "+- ") + + if trimLine == LFSMetaFileIdentifier { + curFileLFSPrefix = true + } + + if curFileLFSPrefix && strings.HasPrefix(trimLine, LFSMetaFileOidPrefix) { + oid := strings.TrimPrefix(trimLine, LFSMetaFileOidPrefix) + + if len(oid) == 64 { + m := &LFSMetaObject{Oid: oid} + count, err := x.Count(m) + + if err == nil && count > 0 { + curFile.IsBin = true + curSection.Lines = nil + break + } + } + } + curFileLinesCount++ lineCount++ @@ -354,6 +376,7 @@ func ParsePatch(maxLines, maxLineCharacteres, maxFiles int, reader io.Reader) (* break } curFileLinesCount = 0 + curFileLFSPrefix = false // Check file diff type and is submodule. for { diff --git a/models/lfs.go b/models/lfs.go index 7eaf6cc007ade..c56426b0cdc84 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -25,6 +25,11 @@ var ( ErrLFSObjectNotExist = errors.New("LFS Meta object does not exist") ) +const ( + LFSMetaFileIdentifier = "version https://git-lfs.github.com/spec/v1" + LFSMetaFileOidPrefix = "oid sha256:" +) + func NewLFSMetaObject(m *LFSMetaObject) (*LFSMetaObject, error) { var err error diff --git a/routers/repo/view.go b/routers/repo/view.go index 2f3bd03d4508c..b92380d897a5a 100644 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -145,10 +145,10 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st //Check for LFS meta file if isTextFile && setting.LFS.StartServer { headString := string(buf) - if strings.HasPrefix(headString, "version https://git-lfs.github.com/spec/v1") { + if strings.HasPrefix(headString, models.LFSMetaFileIdentifier) { splitLines := strings.Split(headString, "\n") if len(splitLines) >= 3 { - oid := strings.TrimPrefix(splitLines[1], "oid sha256:") + oid := strings.TrimPrefix(splitLines[1], models.LFSMetaFileOidPrefix) size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64) if len(oid) == 64 && err == nil { contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath} From d8a019d6dcb0ea1f05041661463d9be3834d3f2f Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Sun, 20 Nov 2016 15:03:28 +0100 Subject: [PATCH 33/37] Show LFS notice if file in commit view is tracked --- models/git_diff.go | 2 ++ templates/repo/diff/box.tmpl | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/models/git_diff.go b/models/git_diff.go index e8b97a04e7a63..f796f286d6d34 100644 --- a/models/git_diff.go +++ b/models/git_diff.go @@ -200,6 +200,7 @@ type DiffFile struct { IsCreated bool IsDeleted bool IsBin bool + IsLFSFile bool IsRenamed bool IsSubmodule bool Sections []*DiffSection @@ -284,6 +285,7 @@ func ParsePatch(maxLines, maxLineCharacteres, maxFiles int, reader io.Reader) (* if err == nil && count > 0 { curFile.IsBin = true + curFile.IsLFSFile = true curSection.Lines = nil break } diff --git a/templates/repo/diff/box.tmpl b/templates/repo/diff/box.tmpl index 4f6c70a328bfb..8ee4a001ce0b0 100644 --- a/templates/repo/diff/box.tmpl +++ b/templates/repo/diff/box.tmpl @@ -66,7 +66,7 @@ - {{.Deletion}} {{end}}
- {{if $file.IsRenamed}}{{$file.OldName}} → {{end}}{{$file.Name}} + {{if $file.IsRenamed}}{{$file.OldName}} → {{end}}{{$file.Name}}{{if .IsLFSFile}} ({{$.i18n.Tr "repo.stored_lfs"}}){{end}} {{if not $file.IsSubmodule}}
{{if $file.IsDeleted}} From c8638c72ec42a28f9dfd1b56b902a1716f1a7a5e Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Sun, 25 Dec 2016 16:48:00 +0100 Subject: [PATCH 34/37] Add notbefore/nbf JWT claim --- cmd/serve.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/serve.go b/cmd/serve.go index 552add5035247..0605bc0868fe9 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -287,11 +287,12 @@ func runServ(c *cli.Context) error { url := fmt.Sprintf("%s%s/%s.git/info/lfs", setting.AppURL, repoUser.Name, repo.Name) - now := time.Now().UTC() + now := time.Now() token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{ "repo": repo.ID, "op": lfsVerb, "exp": now.Add(5 * time.Minute).Unix(), + "nbf": now.Unix(), }) // Sign and get the complete encoded token as a string using the secret From 3f4ed0006f7f8fd9e578dd0d922c497e959821d7 Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Sun, 25 Dec 2016 17:21:20 +0100 Subject: [PATCH 35/37] Correct lint suggestions - comments for structs and functions - Add comments to LFS model - Function comment for GetRandomBytesAsBase64 - LFS server function comments and lint variable suggestion --- models/lfs.go | 21 ++++++++++++++++++++- modules/base/tool.go | 2 +- modules/lfs/server.go | 10 ++++++++-- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/models/lfs.go b/models/lfs.go index c56426b0cdc84..99b8138ab7586 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -6,6 +6,7 @@ import ( "time" ) +// LFSMetaObject stores metadata for LFS tracked files. type LFSMetaObject struct { ID int64 `xorm:"pk autoincr"` Oid string `xorm:"UNIQUE(s) INDEX NOT NULL"` @@ -16,20 +17,31 @@ type LFSMetaObject struct { CreatedUnix int64 } +// LFSTokenResponse defines the JSON structure in which the JWT token is stored. +// This structure is fetched via SSH and passed by the Git LFS client to the server +// endpoint for authorization. type LFSTokenResponse struct { Header map[string]string `json:"header"` Href string `json:"href"` } var ( + // ErrLFSObjectNotExist is returned from lfs models functions in order + // to differentiate between database and missing object errors. ErrLFSObjectNotExist = errors.New("LFS Meta object does not exist") ) const ( + // LFSMetaFileIdentifier is the string appearing at the first line of LFS pointer files. + // https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md LFSMetaFileIdentifier = "version https://git-lfs.github.com/spec/v1" - LFSMetaFileOidPrefix = "oid sha256:" + + // LFSMetaFileOidPrefix appears in LFS pointer files on a line before the sha256 hash. + LFSMetaFileOidPrefix = "oid sha256:" ) +// NewLFSMetaObject stores a given populated LFSMetaObject structure in the database +// if it is not already present. func NewLFSMetaObject(m *LFSMetaObject) (*LFSMetaObject, error) { var err error @@ -56,6 +68,9 @@ func NewLFSMetaObject(m *LFSMetaObject) (*LFSMetaObject, error) { return m, sess.Commit() } +// GetLFSMetaObjectByOid selects a LFSMetaObject entry from database by its OID. +// It may return ErrLFSObjectNotExist or a database error. If the error is nil, +// the returned pointer is a valid LFSMetaObject. func GetLFSMetaObjectByOid(oid string) (*LFSMetaObject, error) { if len(oid) == 0 { return nil, ErrLFSObjectNotExist @@ -71,6 +86,8 @@ func GetLFSMetaObjectByOid(oid string) (*LFSMetaObject, error) { return m, nil } +// RemoveLFSMetaObjectByOid removes a LFSMetaObject entry from database by its OID. +// It may return ErrLFSObjectNotExist or a database error. func RemoveLFSMetaObjectByOid(oid string) error { if len(oid) == 0 { return ErrLFSObjectNotExist @@ -91,10 +108,12 @@ func RemoveLFSMetaObjectByOid(oid string) error { return sess.Commit() } +// BeforeInsert sets the time at which the LFSMetaObject was created. func (m *LFSMetaObject) BeforeInsert() { m.CreatedUnix = time.Now().Unix() } +// AfterSet stores the LFSMetaObject creation time in the database as local time. func (m *LFSMetaObject) AfterSet(colName string, _ xorm.Cell) { switch colName { case "created_unix": diff --git a/modules/base/tool.go b/modules/base/tool.go index 1f02b73bbeb24..f4249f6d6dd39 100644 --- a/modules/base/tool.go +++ b/modules/base/tool.go @@ -104,7 +104,7 @@ func GetRandomString(n int) (string, error) { return string(buffer), nil } -// GetRandomBytes generates a random base64 string from n bytes +// GetRandomBytesAsBase64 generates a random base64 string from n bytes func GetRandomBytesAsBase64(n int) string { bytes := make([]byte, 32) _, err := io.ReadFull(rand.Reader, bytes) diff --git a/modules/lfs/server.go b/modules/lfs/server.go index 804e8330fceb8..f82cb70364fd6 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -35,12 +35,16 @@ type RequestVars struct { Authorization string } +// BatchVars contains multiple RequestVars processed in one batch operation. +// https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md type BatchVars struct { Transfers []string `json:"transfers,omitempty"` Operation string `json:"operation"` Objects []*RequestVars `json:"objects"` } +// BatchResponse contains multiple object metadata Representation structures +// for use with the batch API. type BatchResponse struct { Transfer string `json:"transfer,omitempty"` Objects []*Representation `json:"objects"` @@ -54,6 +58,7 @@ type Representation struct { Error *ObjectError `json:"error,omitempty"` } +// ObjectError defines the JSON structure returned to the client in case of an error type ObjectError struct { Code int `json:"code"` Message string `json:"message"` @@ -71,6 +76,7 @@ type link struct { ExpiresAt time.Time `json:"expires_at,omitempty"` } +// ObjectOidHandler is the main request routing entry point into LFS server functions func ObjectOidHandler(ctx *context.Context) { if !setting.LFS.StartServer { @@ -525,12 +531,12 @@ func authenticateToken(repository *models.Repository, authorization string, requ return false } - repoId, ok := claims["repo"].(float64) + repoID, ok := claims["repo"].(float64) if !ok { return false } - if repository.ID != int64(repoId) { + if repository.ID != int64(repoID) { return false } From 1817c86c4d46c7de8067779bc6c1245c466199df Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Sun, 25 Dec 2016 20:13:27 +0100 Subject: [PATCH 36/37] Move secret generation code out of conditional Ensures no LFS code may run with an empty secret --- modules/setting/setting.go | 65 ++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 125fa1cde2b37..d91cc5770ea20 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -599,10 +599,43 @@ please consider changing to GITEA_CUSTOM`) } if LFS.StartServer { + if err := os.MkdirAll(LFS.ContentPath, 0700); err != nil { log.Fatal(4, "Fail to create '%s': %v", LFS.ContentPath, err) } + LFS.JWTSecretBytes = make([]byte, 32) + n, err := base64.RawURLEncoding.Decode(LFS.JWTSecretBytes, []byte(LFS.JWTSecretBase64)) + + if err != nil || n != 32 { + //Generate new secret and save to config + + _, err := io.ReadFull(rand.Reader, LFS.JWTSecretBytes) + + if err != nil { + log.Fatal(4, "Error reading random bytes: %s", err) + } + + LFS.JWTSecretBase64 = base64.RawURLEncoding.EncodeToString(LFS.JWTSecretBytes) + + // Save secret + cfg := ini.Empty() + if com.IsFile(CustomConf) { + // Keeps custom settings if there is already something. + if err := cfg.Append(CustomConf); err != nil { + log.Error(4, "Fail to load custom conf '%s': %v", CustomConf, err) + } + } + + cfg.Section("server").Key("LFS_JWT_SECRET").SetValue(LFS.JWTSecretBase64) + + os.MkdirAll(filepath.Dir(CustomConf), os.ModePerm) + if err := cfg.SaveTo(CustomConf); err != nil { + log.Fatal(4, "Error saving generated JWT Secret to custom config: %v", err) + return + } + } + //Disable LFS client hooks if installed for the current OS user //Needs at least git v2.1.2 @@ -629,6 +662,7 @@ please consider changing to GITEA_CUSTOM`) if !((majorVersion > 2) || (majorVersion == 2 && minorVersion > 1) || (majorVersion == 2 && minorVersion == 1 && revisionVersion >= 2)) { + LFS.StartServer = false log.Error(4, "LFS server support needs at least Git v2.1.2") } else { @@ -636,37 +670,6 @@ please consider changing to GITEA_CUSTOM`) git.GlobalCommandArgs = append(git.GlobalCommandArgs, "-c", "filter.lfs.required=", "-c", "filter.lfs.smudge=", "-c", "filter.lfs.clean=") - LFS.JWTSecretBytes = make([]byte, 32) - n, err := base64.RawURLEncoding.Decode(LFS.JWTSecretBytes, []byte(LFS.JWTSecretBase64)) - - if err != nil || n != 32 { - //Generate new secret and save to config - - _, err := io.ReadFull(rand.Reader, LFS.JWTSecretBytes) - - if err != nil { - log.Fatal(4, "Error reading random bytes: %s", err) - } - - LFS.JWTSecretBase64 = base64.RawURLEncoding.EncodeToString(LFS.JWTSecretBytes) - - // Save secret - cfg := ini.Empty() - if com.IsFile(CustomConf) { - // Keeps custom settings if there is already something. - if err := cfg.Append(CustomConf); err != nil { - log.Error(4, "Fail to load custom conf '%s': %v", CustomConf, err) - } - } - - cfg.Section("server").Key("LFS_JWT_SECRET").SetValue(LFS.JWTSecretBase64) - - os.MkdirAll(filepath.Dir(CustomConf), os.ModePerm) - if err := cfg.SaveTo(CustomConf); err != nil { - log.Fatal(4, "Error saving generated JWT Secret to custom config: %v", err) - return - } - } } } From 6a38a8bf3a25217f9c4a0b55383c3319fa364503 Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Sun, 25 Dec 2016 20:17:19 +0100 Subject: [PATCH 37/37] Do not hand out JWT tokens if LFS server support is disabled --- cmd/serve.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/cmd/serve.go b/cmd/serve.go index 0605bc0868fe9..8e498faba73b4 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -167,10 +167,17 @@ func runServ(c *cli.Context) error { verb, args := parseCmd(cmd) var lfsVerb string - if verb == lfsAuthenticateVerb && strings.Contains(args, " ") { - argsSplit := strings.SplitN(args, " ", 2) - args = strings.TrimSpace(argsSplit[0]) - lfsVerb = strings.TrimSpace(argsSplit[1]) + if verb == lfsAuthenticateVerb { + + if !setting.LFS.StartServer { + fail("Unknown git command", "LFS authentication request over SSH denied, LFS support is disabled") + } + + if strings.Contains(args, " ") { + argsSplit := strings.SplitN(args, " ", 2) + args = strings.TrimSpace(argsSplit[0]) + lfsVerb = strings.TrimSpace(argsSplit[1]) + } } repoPath := strings.ToLower(strings.Trim(args, "'"))