diff options
| -rw-r--r-- | container.go | 22 | ||||
| -rw-r--r-- | container_iterator.go | 2 | ||||
| -rw-r--r-- | iterator.go | 6 | ||||
| -rw-r--r-- | object_iterator.go | 212 | ||||
| -rw-r--r-- | object_iterator_test.go | 184 | ||||
| -rwxr-xr-x | util/render_template.go | 2 |
6 files changed, 426 insertions, 2 deletions
diff --git a/container.go b/container.go index f075d15..1e30eff 100644 --- a/container.go +++ b/container.go @@ -175,3 +175,25 @@ func (c *Container) EnsureExists() (*Container, error) { }.Do(c.a.client) return c, err } + +//Objects returns an ObjectIterator that lists the objects in this +//container. The most common use case is: +// +// objects, err := container.Objects().Collect() +// +//You can extend this by configuring the iterator before collecting the results: +// +// iter := container.Objects() +// iter.Prefix = "test-" +// objects, err := iter.Collect() +// +//Or you can use a different iteration method: +// +// err := container.Objects().ForeachDetailed(func (info ObjectInfo) error { +// log.Printf("object %s is %d bytes large!\n", +// info.Object.Name(), info.SizeBytes) +// }) +// +func (c *Container) Objects() *ObjectIterator { + return &ObjectIterator{Container: c} +} diff --git a/container_iterator.go b/container_iterator.go index 8c04be6..c428335 100644 --- a/container_iterator.go +++ b/container_iterator.go @@ -37,7 +37,7 @@ type ContainerInfo struct { //ContainerIterator iterates over the accounts in a container. It is typically //constructed with the Account.Containers() method. For example: // -// //either this... +// //either this... // iter := account.Containers() // iter.Prefix = "test-" // containers, err := iter.Collect() diff --git a/iterator.go b/iterator.go index af01bbe..aebeb91 100644 --- a/iterator.go +++ b/iterator.go @@ -40,6 +40,12 @@ func (i ContainerIterator) getPrefix() string { return i.Prefix } func (i ContainerIterator) getHeaders() map[string]string { return i.Headers } func (i ContainerIterator) getOptions() *RequestOptions { return i.Options } +func (i ObjectIterator) getAccount() *Account { return i.Container.Account() } +func (i ObjectIterator) getContainerName() string { return i.Container.Name() } +func (i ObjectIterator) getPrefix() string { return i.Prefix } +func (i ObjectIterator) getHeaders() map[string]string { return i.Headers } +func (i ObjectIterator) getOptions() *RequestOptions { return i.Options } + //iteratorBase provides shared behavior for ContainerIterator and ObjectIterator. type iteratorBase struct { i iteratorInterface diff --git a/object_iterator.go b/object_iterator.go new file mode 100644 index 0000000..67f3138 --- /dev/null +++ b/object_iterator.go @@ -0,0 +1,212 @@ +/******************************************************************************* +* +* Copyright 2018 Stefan Majewsky <majewsky@gmx.net> +* +* This program is free software: you can redistribute it and/or modify it under +* the terms of the GNU General Public License as published by the Free Software +* Foundation, either version 3 of the License, or (at your option) any later +* version. +* +* This program is distributed in the hope that it will be useful, but WITHOUT ANY +* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +* A PARTICULAR PURPOSE. See the GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License along with +* this program. If not, see <http://www.gnu.org/licenses/>. +* +*******************************************************************************/ + +package schwift + +import ( + "fmt" + "time" +) + +//ObjectInfo is a result type returned by ObjectIterator for detailed +//object listings. The metadata in this type is a subset of Object.Headers(), +//but since it is returned as part of the detailed object listing, it can be +//obtained without making additional HEAD requests on the object(s). +type ObjectInfo struct { + Object *Object + SizeBytes uint64 + ContentType string + Etag string + LastModified time.Time +} + +//ObjectIterator iterates over the objects in a container. It is typically +//constructed with the Container.Objects() method. For example: +// +// //either this... +// iter := container.Objects() +// iter.Prefix = "test-" +// objects, err := iter.Collect() +// +// //...or this +// objects, err := schwift.ObjectIterator{ +// Container: container, +// Prefix: "test-", +// }.Collect() +// +//When listing objects via a GET request on the container, you can choose to +//receive object names only (via the methods without the "Detailed" suffix), +//or object names plus some basic metadata fields (via the methods with the +//"Detailed" suffix). See struct ObjectInfo for which metadata is returned. +// +//To obtain any other metadata, you can call Object.Headers() on the result +//object, but this will issue a separate HEAD request for each object. +// +//Use the "Detailed" methods only when you can use the extra metadata in struct +//ObjectInfo; detailed GET requests are more expensive than simple ones that +//return only object names. +type ObjectIterator struct { + Container *Container + //When Prefix is set, only objects whose name starts with this string are + //returned. + Prefix string + //Headers may contain additional headers to include with the GET request. + Headers map[string]string + //Options may contain additional query parameters for the GET request. + Options *RequestOptions + + base *iteratorBase +} + +func (i *ObjectIterator) getBase() *iteratorBase { + if i.base == nil { + i.base = &iteratorBase{i: i} + } + return i.base +} + +//NextPage queries Swift for the next page of object names. If limit is +//>= 0, not more than that object names will be returned at once. Note +//that the server also has a limit for how many objects to list in one +//request; the lower limit wins. +// +//The end of the object listing is reached when an empty list is returned. +// +//This method offers maximal flexibility, but most users will prefer the +//simpler interfaces offered by Collect() and Foreach(). +func (i *ObjectIterator) NextPage(limit int) ([]*Object, error) { + names, err := i.getBase().nextPage(limit) + if err != nil { + return nil, err + } + + result := make([]*Object, len(names)) + for idx, name := range names { + result[idx] = i.Container.Object(name) + } + return result, nil +} + +//NextPageDetailed is like NextPage, but includes basic metadata. +func (i *ObjectIterator) NextPageDetailed(limit int) ([]ObjectInfo, error) { + b := i.getBase() + + var document []struct { + SizeBytes uint64 `json:"bytes"` + ContentType string `json:"content_type"` + Etag string `json:"hash"` + LastModifiedStr string `json:"last_modified"` + Name string `json:"name"` + } + err := b.nextPageDetailed(limit, &document) + if err != nil { + return nil, err + } + if len(document) == 0 { + b.setMarker("") //indicate EOF to iteratorBase + return nil, nil + } + + result := make([]ObjectInfo, len(document)) + for idx, data := range document { + result[idx].Object = i.Container.Object(data.Name) + result[idx].ContentType = data.ContentType + result[idx].Etag = data.Etag + result[idx].SizeBytes = data.SizeBytes + result[idx].LastModified, err = time.Parse(time.RFC3339Nano, data.LastModifiedStr+"Z") + if err != nil { + //this error is sufficiently obscure that we don't need to expose a type for it + return nil, fmt.Errorf("Bad field objects[%d].last_modified: %s", idx, err.Error()) + } + } + + b.setMarker(result[len(result)-1].Object.Name()) + return result, nil +} + +//Foreach lists the object names matching this iterator and calls the +//callback once for every object. Iteration is aborted when a GET request fails, +//or when the callback returns a non-nil error. +func (i *ObjectIterator) Foreach(callback func(*Object) error) error { + for { + objects, err := i.NextPage(-1) + if err != nil { + return err + } + if len(objects) == 0 { + return nil //EOF + } + for _, o := range objects { + err := callback(o) + if err != nil { + return err + } + } + } +} + +//ForeachDetailed is like Foreach, but includes basic metadata. +func (i *ObjectIterator) ForeachDetailed(callback func(ObjectInfo) error) error { + for { + infos, err := i.NextPageDetailed(-1) + if err != nil { + return err + } + if len(infos) == 0 { + return nil //EOF + } + for _, ci := range infos { + err := callback(ci) + if err != nil { + return err + } + } + } +} + +//Collect lists all object names matching this iterator. For large sets of +//objects that cannot be retrieved at once, Collect handles paging behind +//the scenes. The return value is always the complete set of objects. +func (i *ObjectIterator) Collect() ([]*Object, error) { + var result []*Object + for { + objects, err := i.NextPage(-1) + if err != nil { + return nil, err + } + if len(objects) == 0 { + return result, nil //EOF + } + result = append(result, objects...) + } +} + +//CollectDetailed is like Collect, but includes basic metadata. +func (i *ObjectIterator) CollectDetailed() ([]ObjectInfo, error) { + var result []ObjectInfo + for { + infos, err := i.NextPageDetailed(-1) + if err != nil { + return nil, err + } + if len(infos) == 0 { + return result, nil //EOF + } + result = append(result, infos...) + } +} diff --git a/object_iterator_test.go b/object_iterator_test.go new file mode 100644 index 0000000..4a2387d --- /dev/null +++ b/object_iterator_test.go @@ -0,0 +1,184 @@ +/****************************************************************************** +* +* Copyright 2018 Stefan Majewsky <majewsky@gmx.net> +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +******************************************************************************/ + +package schwift + +import ( + "bytes" + "crypto/md5" + "encoding/hex" + "fmt" + "testing" +) + +var objectExampleContent = []byte(`{"message":"Hello World!"}`) +var objectExampleContentEtag = etagOf(objectExampleContent) + +func etagOf(buf []byte) string { + hash := md5.Sum(buf) + return hex.EncodeToString(hash[:]) +} + +func TestObjectIterator(t *testing.T) { + testWithContainer(t, func(c *Container) { + oname := func(idx int) string { + return fmt.Sprintf("schwift-test-listing%d", idx) + } + + //create test objects that can be listed + for idx := 1; idx <= 4; idx++ { + hdr := make(ObjectHeaders) + hdr.ContentType().Set("application/json") + err := c.Object(oname(idx)).Upload(bytes.NewReader(objectExampleContent), hdr, nil) + expectSuccess(t, err) + } + + //test iteration with empty last page + iter := c.Objects() + iter.Prefix = "schwift-test-listing" + os, err := iter.NextPage(2) + expectSuccess(t, err) + expectObjectNames(t, os, oname(1), oname(2)) + os, err = iter.NextPage(2) + expectSuccess(t, err) + expectObjectNames(t, os, oname(3), oname(4)) + os, err = iter.NextPage(2) + expectSuccess(t, err) + expectObjectNames(t, os) + os, err = iter.NextPage(2) + expectSuccess(t, err) + expectObjectNames(t, os) + + //test iteration with partial last page + iter = c.Objects() + iter.Prefix = "schwift-test-listing" + os, err = iter.NextPage(3) + expectSuccess(t, err) + expectObjectNames(t, os, oname(1), oname(2), oname(3)) + os, err = iter.NextPage(3) + expectSuccess(t, err) + expectObjectNames(t, os, oname(4)) + os, err = iter.NextPage(4) + expectSuccess(t, err) + expectObjectNames(t, os) + + //test detailed iteration + iter = c.Objects() + iter.Prefix = "schwift-test-listing" + ois, err := iter.NextPageDetailed(2) + expectSuccess(t, err) + expectObjectInfos(t, ois, oname(1), oname(2)) + ois, err = iter.NextPageDetailed(3) + expectSuccess(t, err) + expectObjectInfos(t, ois, oname(3), oname(4)) + ois, err = iter.NextPageDetailed(3) + expectSuccess(t, err) + expectObjectInfos(t, ois) + ois, err = iter.NextPageDetailed(3) + expectSuccess(t, err) + expectObjectInfos(t, ois) + + //test Foreach + iter = c.Objects() + iter.Prefix = "schwift-test-listing" + idx := 0 + expectSuccess(t, iter.Foreach(func(o *Object) error { + idx++ + expectString(t, o.Name(), oname(idx)) + return nil + })) + expectInt(t, idx, 4) + + //test ForeachDetailed + iter = c.Objects() + iter.Prefix = "schwift-test-listing" + idx = 0 + expectSuccess(t, iter.ForeachDetailed(func(info ObjectInfo) error { + idx++ + expectString(t, info.Object.Name(), oname(idx)) + return nil + })) + expectInt(t, idx, 4) + + //test Collect + iter = c.Objects() + iter.Prefix = "schwift-test-listing" + os, err = iter.Collect() + expectSuccess(t, err) + expectObjectNames(t, os, oname(1), oname(2), oname(3), oname(4)) + + //test CollectDetailed + iter = c.Objects() + iter.Prefix = "schwift-test-listing" + ois, err = iter.CollectDetailed() + expectSuccess(t, err) + expectObjectInfos(t, ois, oname(1), oname(2), oname(3), oname(4)) + + //cleanup + iter = c.Objects() + iter.Prefix = "schwift-test-listing" + expectSuccess(t, iter.Foreach(func(o *Object) error { + return o.Delete(nil, nil) + })) + }) +} + +func expectObjectNames(t *testing.T, actualObjects []*Object, expectedNames ...string) { + t.Helper() + if len(actualObjects) != len(expectedNames) { + t.Errorf("expected %d objects, got %d objects", + len(expectedNames), len(actualObjects)) + return + } + for idx, c := range actualObjects { + if c.Name() != expectedNames[idx] { + t.Errorf("expected objects[%d].Name() == %q, got %q", + idx, expectedNames[idx], c.Name()) + } + } +} + +func expectObjectInfos(t *testing.T, actualInfos []ObjectInfo, expectedNames ...string) { + t.Helper() + if len(actualInfos) != len(expectedNames) { + t.Errorf("expected %d objects, got %d objects", + len(expectedNames), len(actualInfos)) + return + } + for idx, info := range actualInfos { + if info.Object.Name() != expectedNames[idx] { + t.Errorf("expected objects[%d].Name() == %q, got %q", + idx, expectedNames[idx], info.Object.Name()) + } + if info.SizeBytes != uint64(len(objectExampleContent)) { + t.Errorf("expected objects[%d] sizeBytes == %d, got %d", + idx, len(objectExampleContent), info.SizeBytes) + } + if info.ContentType != "application/json" { + t.Errorf(`expected objects[%d] contentType == "application/json", got %q`, + idx, info.ContentType) + } + if info.Etag != objectExampleContentEtag { + t.Errorf("expected objects[%d] etag == %q, got %q", + idx, objectExampleContentEtag, info.Etag) + } + if info.LastModified.IsZero() { + t.Errorf("objects[%d].LastModified is zero", idx) + } + } +} diff --git a/util/render_template.go b/util/render_template.go index 1db8840..2898623 100755 --- a/util/render_template.go +++ b/util/render_template.go @@ -23,10 +23,10 @@ package main import ( "encoding/json" "fmt" - "html/template" "io/ioutil" "os" "strings" + "text/template" ) func main() { |