Make hierarchy creation the only mode for blobstore. Cache a 'good' directory structure with a 'blobstore' file in the blobstore directory.

pkg/blobstore/blobstore.go

@@ -19,7 +19,6 @@ import (
 type BlobStore struct {
 	dir     string
 	lockfn  string
-	makeall bool
 }
 
 const (
@@ -42,7 +41,7 @@ var (
 // Open an existing, or create a new BlobStore at path.
 // Path must point to a directory, and must already exist.
 // See NewBlobStore for more information.
-func Open(path string, makeall bool) (err error) {
+func Open(path string) (err error) {
 	defaultMutex.Lock()
 	defer defaultMutex.Unlock()
 
@@ -50,7 +49,7 @@ func Open(path string, makeall bool) (err error) {
 		panic("Default BlobStore already open")
 	}
 
-	defaultStore, err = NewBlobStore(path, makeall)
+	defaultStore, err = NewBlobStore(path)
 	return
 }
 
@@ -84,15 +83,7 @@ func Put(buf []byte) (key string, err error) {
 
 // Open an existing, or create a new BlobStore residing at path.
 // Path must point to a directory, and must already exist.
-//
-// The makeall argument determines whether the BlobStore should
-// create all possible blob-container directories a priori.
-// This can take up a bit of disk space since the metadata for
-// those directories can take up a lot of space. However, tt saves
-// some I/O operations when writing blobs. (Since the BlobStore
-// knows that all directories will exist, it does not need to check
-// whether they do, and create them if they do not.).
-func NewBlobStore(path string, makeall bool) (bs *BlobStore, err error) {
+func NewBlobStore(path string) (bs *BlobStore, err error) {
 	// Does the directory exist?
 	dir, err := os.Open(path)
 	if err != nil {
@@ -115,7 +106,21 @@ func NewBlobStore(path string, makeall bool) (bs *BlobStore, err error) {
 		}
 	}()
 
-	if makeall {
+	dirStructureExists := true
+	// Check whether a 'blobstore' file exists in the directory.
+	// The existence of the file signals that the directory already
+	// has the correct hierarchy structure.
+	bsf, err := os.Open(filepath.Join(path, "blobstore"))
+	if err != nil {
+		if e, ok := err.(*os.PathError); ok {
+			if e.Err == os.ENOENT {
+				dirStructureExists = false
+			}
+		}
+	}
+	bsf.Close()
+
+	if !dirStructureExists {
 		for i := 0; i < 256; i++ {
 			outer := filepath.Join(path, hex.EncodeToString([]byte{byte(i)}))
 			err = os.Mkdir(outer, 0700)
@@ -158,12 +163,19 @@ func NewBlobStore(path string, makeall bool) (bs *BlobStore, err error) {
 				}
 			}
 		}
+
+		// Add a blobstore file to signal that a correct directory
+		// structure exists for this blobstore.
+		bsf, err = os.Create(filepath.Join(path, "blobstore"))
+		if err != nil {
+			return nil, err
+		}
+		bsf.Close()
 	}
 
 	bs = &BlobStore{
 		dir:     path,
 		lockfn:  lockfn,
-		makeall: makeall,
 	}
 	return bs, nil
 }
@@ -270,14 +282,6 @@ func (bs *BlobStore) Put(buf []byte) (key string, err error) {
 		}
 	}
 
-	if !bs.makeall {
-		// Make sure the leading directories exist...
-		err = os.MkdirAll(filepath.ToSlash(blobdir), 0700)
-		if err != nil {
-			return
-		}
-	}
-
 	// Create a temporary file to write to. Once we're done, we
 	// can atomically rename the file to the correct key.
 	file, err = ioutil.TempFile(blobdir, rest)

pkg/blobstore/blobstore_test.go

@@ -22,7 +22,7 @@ func TestMakeAllCreateAll(t *testing.T) {
 	}
 	defer os.RemoveAll(dir)
 
-	bs, err := NewBlobStore(dir, true)
+	bs, err := NewBlobStore(dir)
 	if err != nil {
 		t.Error(err)
 		return
@@ -35,7 +35,7 @@ func TestMakeAllCreateAll(t *testing.T) {
 			dirname := filepath.Join(dir, hex.EncodeToString([]byte{byte(i)}), hex.EncodeToString([]byte{byte(j)}))
 			fi, err := os.Stat(dirname)
 			if err != nil {
-				t.Error(err)
+				t.Fatal(err)
 			}
 			if !fi.IsDirectory() {
 				t.Errorf("Not a directory")
@@ -57,7 +57,7 @@ func TestAllInvalidFiles(t *testing.T) {
 		t.Error(err)
 	}
 
-	_, err = NewBlobStore(dir, true)
+	_, err = NewBlobStore(dir)
 	if err == ErrBadFile {
 		// Success
 	} else if err != nil {
@@ -85,7 +85,7 @@ func TestAllInvalidFilesLevel2(t *testing.T) {
 		t.Error(err)
 	}
 
-	_, err = NewBlobStore(dir, true)
+	_, err = NewBlobStore(dir)
 	if err == ErrBadFile {
 		// Success
 	} else if err != nil {
@@ -103,7 +103,7 @@ func TestStoreRetrieve(t *testing.T) {
 	}
 	defer os.RemoveAll(dir)
 
-	bs, err := NewBlobStore(dir, false)
+	bs, err := NewBlobStore(dir)
 	if err != nil {
 		t.Error(err)
 		return
@@ -136,7 +136,7 @@ func TestReadNonExistantKey(t *testing.T) {
 	}
 	defer os.RemoveAll(dir)
 
-	bs, err := NewBlobStore(dir, false)
+	bs, err := NewBlobStore(dir)
 	if err != nil {
 		t.Error(err)
 		return
@@ -160,7 +160,7 @@ func TestReadInvalidKeyLength(t *testing.T) {
 	}
 	defer os.RemoveAll(dir)
 
-	bs, err := NewBlobStore(dir, false)
+	bs, err := NewBlobStore(dir)
 	if err != nil {
 		t.Error(err)
 		return
@@ -187,7 +187,7 @@ func TestReadInvalidKeyNonHex(t *testing.T) {
 	}
 	defer os.RemoveAll(dir)
 
-	bs, err := NewBlobStore(dir, false)
+	bs, err := NewBlobStore(dir)
 	if err != nil {
 		t.Error(err)
 		return
@@ -214,7 +214,7 @@ func TestDefaultBlobStore(t *testing.T) {
 	}
 	defer os.RemoveAll(dir)
 
-	err = Open(dir, false)
+	err = Open(dir)
 	if err != nil {
 		t.Error(err)
 	}