add support for pseudo-directories to ObjectIterator

3 files changed, 125 insertions, 29 deletions

diff --git a/iterator.go b/iterator.go
index 04fb492..641e28d 100644
--- a/iterator.go
+++ b/iterator.go
@@ -30,6 +30,7 @@ import (
 type iteratorInterface interface {
 	getAccount() *Account
 	getContainerName() string
+	getDelimiter() string
 	getPrefix() string
 	getOptions() *RequestOptions
 	//putHeader initializes the AccountHeaders/ContainerHeaders field of the
@@ -39,6 +40,7 @@ type iteratorInterface interface {
 
 func (i ContainerIterator) getAccount() *Account        { return i.Account }
 func (i ContainerIterator) getContainerName() string    { return "" }
+func (i ContainerIterator) getDelimiter() string        { return "" }
 func (i ContainerIterator) getPrefix() string           { return i.Prefix }
 func (i ContainerIterator) getOptions() *RequestOptions { return i.Options }
 
@@ -53,6 +55,7 @@ func (i ContainerIterator) putHeader(hdr http.Header) error {
 
 func (i ObjectIterator) getAccount() *Account        { return i.Container.Account() }
 func (i ObjectIterator) getContainerName() string    { return i.Container.Name() }
+func (i ObjectIterator) getDelimiter() string        { return i.Delimiter }
 func (i ObjectIterator) getPrefix() string           { return i.Prefix }
 func (i ObjectIterator) getOptions() *RequestOptions { return i.Options }
 
@@ -79,6 +82,9 @@ func (b *iteratorBase) request(limit int, detailed bool) Request {
 		Options:       cloneRequestOptions(b.i.getOptions(), nil),
 	}
 
+	if delimiter := b.i.getDelimiter(); delimiter != "" {
+		r.Options.Values.Set("delimiter", delimiter)
+	}
 	if prefix := b.i.getPrefix(); prefix != "" {
 		r.Options.Values.Set("prefix", prefix)
 	}
diff --git a/object_iterator.go b/object_iterator.go
index 3fb5563..6231292 100644
--- a/object_iterator.go
+++ b/object_iterator.go
@@ -33,6 +33,11 @@ type ObjectInfo struct {
 	ContentType  string
 	Etag         string
 	LastModified time.Time
+	//If the ObjectInfo refers to an actual object, then SubDirectory is empty.
+	//If the ObjectInfo refers to a pseudo-directory, then SubDirectory contains
+	//the path of the pseudo-directory and all other fields are nil/zero/empty.
+	//Pseudo-directories will only be reported for ObjectIterator.Delimiter != "".
+	SubDirectory string
 }
 
 //ObjectIterator iterates over the objects in a container. It is typically
@@ -60,16 +65,22 @@ type ObjectInfo struct {
 //Use the "Detailed" methods only when you use the extra metadata in struct
 //ObjectInfo; detailed GET requests are more expensive than simple ones that
 //return only object names.
+//
+//Note that, when Delimiter is set, instances of *Object that you receive from
+//the iterator may refer to a pseudo-directory instead of an actual object, in
+//which case Exists() will return false.
 type ObjectIterator struct {
 	Container *Container
 	//When Prefix is set, only objects whose name starts with this string are
 	//returned.
 	Prefix string
+	//When Delimiter is set, objects whose name contains this string (after the
+	//prefix, if any) will be condensed into pseudo-directories in the result.
+	//See documentation for Swift for details.
+	Delimiter string
 	//Options may contain additional headers and query parameters for the GET request.
 	Options *RequestOptions
 
-	//TODO: Delimiter field (and check if other stuff is missing)
-
 	base *iteratorBase
 }
 
@@ -107,11 +118,14 @@ func (i *ObjectIterator) NextPageDetailed(limit int) ([]ObjectInfo, error) {
 	b := i.getBase()
 
 	var document []struct {
+		//either all of this:
 		SizeBytes       uint64 `json:"bytes"`
 		ContentType     string `json:"content_type"`
 		Etag            string `json:"hash"`
 		LastModifiedStr string `json:"last_modified"`
 		Name            string `json:"name"`
+		//or just this:
+		Subdir string `json:"subdir"`
 	}
 	err := b.nextPageDetailed(limit, &document)
 	if err != nil {
@@ -123,19 +137,26 @@ func (i *ObjectIterator) NextPageDetailed(limit int) ([]ObjectInfo, error) {
 	}
 
 	result := make([]ObjectInfo, len(document))
+	marker := ""
 	for idx, data := range document {
-		result[idx].Object = i.Container.Object(data.Name)
-		result[idx].ContentType = data.ContentType
-		result[idx].Etag = data.Etag
-		result[idx].SizeBytes = data.SizeBytes
-		result[idx].LastModified, err = time.Parse(time.RFC3339Nano, data.LastModifiedStr+"Z")
-		if err != nil {
-			//this error is sufficiently obscure that we don't need to expose a type for it
-			return nil, fmt.Errorf("Bad field objects[%d].last_modified: %s", idx, err.Error())
+		if data.Subdir == "" {
+			marker = data.Name
+			result[idx].Object = i.Container.Object(data.Name)
+			result[idx].ContentType = data.ContentType
+			result[idx].Etag = data.Etag
+			result[idx].SizeBytes = data.SizeBytes
+			result[idx].LastModified, err = time.Parse(time.RFC3339Nano, data.LastModifiedStr+"Z")
+			if err != nil {
+				//this error is sufficiently obscure that we don't need to expose a type for it
+				return nil, fmt.Errorf("Bad field objects[%d].last_modified: %s", idx, err.Error())
+			}
+		} else {
+			marker = data.Subdir
+			result[idx].SubDirectory = data.Subdir
 		}
 	}
 
-	b.setMarker(result[len(result)-1].Object.Name())
+	b.setMarker(marker)
 	return result, nil
 }
 
diff --git a/tests/object_iterator_test.go b/tests/object_iterator_test.go
index 9b822cd..e4a9c6e 100644
--- a/tests/object_iterator_test.go
+++ b/tests/object_iterator_test.go
@@ -21,6 +21,7 @@ package tests
 import (
 	"bytes"
 	"fmt"
+	"strings"
 	"testing"
 
 	"github.com/majewsky/schwift"
@@ -130,6 +131,55 @@ func TestObjectIterator(t *testing.T) {
 	})
 }
 
+func TestPseudoDirectories(t *testing.T) {
+	testWithContainer(t, func(c *schwift.Container) {
+		//create test objects that can be listed
+		objectNames := []string{
+			"foo/1",
+			"foo/2",
+			"foo/3",
+			"foo/bar",
+			"foo/bar/1",
+			"foo/bar/2",
+			"foo/bar/3",
+		}
+		for _, name := range objectNames {
+			hdr := schwift.NewObjectHeaders()
+			hdr.ContentType().Set("application/json")
+			err := c.Object(name).Upload(bytes.NewReader(objectExampleContent), nil, hdr.ToOpts())
+			expectSuccess(t, err)
+		}
+
+		//test iteration with Delimiter and no Prefix
+		iter := c.Objects()
+		iter.Delimiter = "/"
+		os, err := iter.Collect()
+		expectSuccess(t, err)
+		expectObjectNames(t, os, "foo/")
+
+		iter = c.Objects()
+		iter.Delimiter = "/"
+		ois, err := iter.CollectDetailed()
+		expectSuccess(t, err)
+		expectObjectInfos(t, ois, "subdir:foo/")
+
+		//test iteration with Delimited and Prefix
+		iter = c.Objects()
+		iter.Prefix = "foo/"
+		iter.Delimiter = "/"
+		os, err = iter.Collect()
+		expectSuccess(t, err)
+		expectObjectNames(t, os, "foo/1", "foo/2", "foo/3", "foo/bar", "foo/bar/")
+
+		iter = c.Objects()
+		iter.Prefix = "foo/"
+		iter.Delimiter = "/"
+		ois, err = iter.CollectDetailed()
+		expectSuccess(t, err)
+		expectObjectInfos(t, ois, "foo/1", "foo/2", "foo/3", "foo/bar", "subdir:foo/bar/")
+	})
+}
+
 func expectContainerHeadersCached(t *testing.T, c *schwift.Container) {
 	requestCountBefore := c.Account().Backend().(*RequestCountingBackend).Count
 	_, err := c.Headers()
@@ -165,24 +215,43 @@ func expectObjectInfos(t *testing.T, actualInfos []schwift.ObjectInfo, expectedN
 		return
 	}
 	for idx, info := range actualInfos {
-		if info.Object.Name() != expectedNames[idx] {
-			t.Errorf("expected objects[%d].Name() == %q, got %q",
-				idx, expectedNames[idx], info.Object.Name())
-		}
-		if info.SizeBytes != uint64(len(objectExampleContent)) {
-			t.Errorf("expected objects[%d] sizeBytes == %d, got %d",
-				idx, len(objectExampleContent), info.SizeBytes)
-		}
-		if info.ContentType != "application/json" {
-			t.Errorf(`expected objects[%d] contentType == "application/json", got %q`,
-				idx, info.ContentType)
-		}
-		if info.Etag != objectExampleContentEtag {
-			t.Errorf("expected objects[%d] etag == %q, got %q",
-				idx, objectExampleContentEtag, info.Etag)
-		}
-		if info.LastModified.IsZero() {
-			t.Errorf("objects[%d].LastModified is zero", idx)
+		if strings.HasPrefix(expectedNames[idx], "subdir:") {
+			expectedSubdir := strings.TrimPrefix(expectedNames[idx], "subdir:")
+			if expectedSubdir != info.SubDirectory {
+				t.Errorf("expected objects[%d] subdir = %q, got %q",
+					idx, expectedSubdir, info.SubDirectory)
+			}
+			if info != (schwift.ObjectInfo{SubDirectory: info.SubDirectory}) {
+				t.Errorf("expected objects[%d] to be a subdir, got %#v",
+					idx, info)
+			}
+		} else {
+			if info.SubDirectory != "" {
+				t.Errorf("expected objects[%d] to be an object, got subdir = %q",
+					idx, info.SubDirectory)
+			}
+			if info.Object == nil {
+				t.Errorf("expected objects[%d].Name() == %q, got object == nil",
+					idx, expectedNames[idx])
+			} else if info.Object.Name() != expectedNames[idx] {
+				t.Errorf("expected objects[%d].Name() == %q, got %q",
+					idx, expectedNames[idx], info.Object.Name())
+			}
+			if info.SizeBytes != uint64(len(objectExampleContent)) {
+				t.Errorf("expected objects[%d] sizeBytes == %d, got %d",
+					idx, len(objectExampleContent), info.SizeBytes)
+			}
+			if info.ContentType != "application/json" {
+				t.Errorf(`expected objects[%d] contentType == "application/json", got %q`,
+					idx, info.ContentType)
+			}
+			if info.Etag != objectExampleContentEtag {
+				t.Errorf("expected objects[%d] etag == %q, got %q",
+					idx, objectExampleContentEtag, info.Etag)
+			}
+			if info.LastModified.IsZero() {
+				t.Errorf("objects[%d].LastModified is zero", idx)
+			}
 		}
 	}
 }