From 852d1790aeea304e7b4a56bb3fcf7472847e10d3 Mon Sep 17 00:00:00 2001 From: Mikkel Krautz Date: Tue, 8 Nov 2011 15:38:30 +0100 Subject: [PATCH] Make hierarchy creation the only mode for blobstore. Cache a 'good' directory structure with a 'blobstore' file in the blobstore directory. --- pkg/blobstore/blobstore.go | 48 ++++++++++++++++++--------------- pkg/blobstore/blobstore_test.go | 18 ++++++------- 2 files changed, 35 insertions(+), 31 deletions(-) diff --git a/pkg/blobstore/blobstore.go b/pkg/blobstore/blobstore.go index 28244b8..8e2ac1d 100644 --- a/pkg/blobstore/blobstore.go +++ b/pkg/blobstore/blobstore.go @@ -19,7 +19,6 @@ import ( type BlobStore struct { dir string lockfn string - makeall bool } const ( @@ -42,7 +41,7 @@ var ( // Open an existing, or create a new BlobStore at path. // Path must point to a directory, and must already exist. // See NewBlobStore for more information. -func Open(path string, makeall bool) (err error) { +func Open(path string) (err error) { defaultMutex.Lock() defer defaultMutex.Unlock() @@ -50,7 +49,7 @@ func Open(path string, makeall bool) (err error) { panic("Default BlobStore already open") } - defaultStore, err = NewBlobStore(path, makeall) + defaultStore, err = NewBlobStore(path) return } @@ -84,15 +83,7 @@ func Put(buf []byte) (key string, err error) { // Open an existing, or create a new BlobStore residing at path. // Path must point to a directory, and must already exist. -// -// The makeall argument determines whether the BlobStore should -// create all possible blob-container directories a priori. -// This can take up a bit of disk space since the metadata for -// those directories can take up a lot of space. However, tt saves -// some I/O operations when writing blobs. (Since the BlobStore -// knows that all directories will exist, it does not need to check -// whether they do, and create them if they do not.). -func NewBlobStore(path string, makeall bool) (bs *BlobStore, err error) { +func NewBlobStore(path string) (bs *BlobStore, err error) { // Does the directory exist? dir, err := os.Open(path) if err != nil { @@ -115,7 +106,21 @@ func NewBlobStore(path string, makeall bool) (bs *BlobStore, err error) { } }() - if makeall { + dirStructureExists := true + // Check whether a 'blobstore' file exists in the directory. + // The existence of the file signals that the directory already + // has the correct hierarchy structure. + bsf, err := os.Open(filepath.Join(path, "blobstore")) + if err != nil { + if e, ok := err.(*os.PathError); ok { + if e.Err == os.ENOENT { + dirStructureExists = false + } + } + } + bsf.Close() + + if !dirStructureExists { for i := 0; i < 256; i++ { outer := filepath.Join(path, hex.EncodeToString([]byte{byte(i)})) err = os.Mkdir(outer, 0700) @@ -158,12 +163,19 @@ func NewBlobStore(path string, makeall bool) (bs *BlobStore, err error) { } } } + + // Add a blobstore file to signal that a correct directory + // structure exists for this blobstore. + bsf, err = os.Create(filepath.Join(path, "blobstore")) + if err != nil { + return nil, err + } + bsf.Close() } bs = &BlobStore{ dir: path, lockfn: lockfn, - makeall: makeall, } return bs, nil } @@ -270,14 +282,6 @@ func (bs *BlobStore) Put(buf []byte) (key string, err error) { } } - if !bs.makeall { - // Make sure the leading directories exist... - err = os.MkdirAll(filepath.ToSlash(blobdir), 0700) - if err != nil { - return - } - } - // Create a temporary file to write to. Once we're done, we // can atomically rename the file to the correct key. file, err = ioutil.TempFile(blobdir, rest) diff --git a/pkg/blobstore/blobstore_test.go b/pkg/blobstore/blobstore_test.go index 2a50406..0377497 100644 --- a/pkg/blobstore/blobstore_test.go +++ b/pkg/blobstore/blobstore_test.go @@ -22,7 +22,7 @@ func TestMakeAllCreateAll(t *testing.T) { } defer os.RemoveAll(dir) - bs, err := NewBlobStore(dir, true) + bs, err := NewBlobStore(dir) if err != nil { t.Error(err) return @@ -35,7 +35,7 @@ func TestMakeAllCreateAll(t *testing.T) { dirname := filepath.Join(dir, hex.EncodeToString([]byte{byte(i)}), hex.EncodeToString([]byte{byte(j)})) fi, err := os.Stat(dirname) if err != nil { - t.Error(err) + t.Fatal(err) } if !fi.IsDirectory() { t.Errorf("Not a directory") @@ -57,7 +57,7 @@ func TestAllInvalidFiles(t *testing.T) { t.Error(err) } - _, err = NewBlobStore(dir, true) + _, err = NewBlobStore(dir) if err == ErrBadFile { // Success } else if err != nil { @@ -85,7 +85,7 @@ func TestAllInvalidFilesLevel2(t *testing.T) { t.Error(err) } - _, err = NewBlobStore(dir, true) + _, err = NewBlobStore(dir) if err == ErrBadFile { // Success } else if err != nil { @@ -103,7 +103,7 @@ func TestStoreRetrieve(t *testing.T) { } defer os.RemoveAll(dir) - bs, err := NewBlobStore(dir, false) + bs, err := NewBlobStore(dir) if err != nil { t.Error(err) return @@ -136,7 +136,7 @@ func TestReadNonExistantKey(t *testing.T) { } defer os.RemoveAll(dir) - bs, err := NewBlobStore(dir, false) + bs, err := NewBlobStore(dir) if err != nil { t.Error(err) return @@ -160,7 +160,7 @@ func TestReadInvalidKeyLength(t *testing.T) { } defer os.RemoveAll(dir) - bs, err := NewBlobStore(dir, false) + bs, err := NewBlobStore(dir) if err != nil { t.Error(err) return @@ -187,7 +187,7 @@ func TestReadInvalidKeyNonHex(t *testing.T) { } defer os.RemoveAll(dir) - bs, err := NewBlobStore(dir, false) + bs, err := NewBlobStore(dir) if err != nil { t.Error(err) return @@ -214,7 +214,7 @@ func TestDefaultBlobStore(t *testing.T) { } defer os.RemoveAll(dir) - err = Open(dir, false) + err = Open(dir) if err != nil { t.Error(err) }