/******************************************************************************* * * Copyright 2018 Stefan Majewsky * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU General Public License as published by the Free Software * Foundation, either version 3 of the License, or (at your option) any later * version. * * This program is distributed in the hope that it will be useful, but WITHOUT ANY * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR * A PARTICULAR PURPOSE. See the GNU General Public License for more details. * * You should have received a copy of the GNU General Public License along with * this program. If not, see . * *******************************************************************************/ package schwift import ( "context" "fmt" "regexp" "time" ) // ObjectInfo is a result type returned by ObjectIterator for detailed // object listings. The metadata in this type is a subset of Object.Headers(), // but since it is returned as part of the detailed object listing, it can be // obtained without making additional HEAD requests on the object(s). type ObjectInfo struct { Object *Object SizeBytes uint64 ContentType string Etag string LastModified time.Time // SymlinkTarget is only set for symlinks. SymlinkTarget *Object // If the ObjectInfo refers to an actual object, then SubDirectory is empty. // If the ObjectInfo refers to a pseudo-directory, then SubDirectory contains // the path of the pseudo-directory and all other fields are nil/zero/empty. // Pseudo-directories will only be reported for ObjectIterator.Delimiter != "". SubDirectory string } // ObjectIterator iterates over the objects in a container. It is typically // constructed with the Container.Objects() method. For example: // // //either this... // iter := container.Objects() // iter.Prefix = "test-" // objects, err := iter.Collect() // // //...or this // objects, err := schwift.ObjectIterator{ // Container: container, // Prefix: "test-", // }.Collect() // // When listing objects via a GET request on the container, you can choose to // receive object names only (via the methods without the "Detailed" suffix), // or object names plus some basic metadata fields (via the methods with the // "Detailed" suffix). See struct ObjectInfo for which metadata is returned. // // To obtain any other metadata, you can call Object.Headers() on the result // object, but this will issue a separate HEAD request for each object. // // Use the "Detailed" methods only when you use the extra metadata in struct // ObjectInfo; detailed GET requests are more expensive than simple ones that // return only object names. // // Note that, when Delimiter is set, instances of *Object that you receive from // the iterator may refer to a pseudo-directory instead of an actual object, in // which case Exists() will return false. type ObjectIterator struct { Container *Container // When Prefix is set, only objects whose name starts with this string are // returned. Prefix string // When Delimiter is set, objects whose name contains this string (after the // prefix, if any) will be condensed into pseudo-directories in the result. // See documentation for Swift for details. Delimiter string // Options may contain additional headers and query parameters for the GET request. Options *RequestOptions base *iteratorBase } func (i *ObjectIterator) getBase() *iteratorBase { if i.base == nil { i.base = &iteratorBase{i: i} } return i.base } // NextPage queries Swift for the next page of object names. If limit is // >= 0, not more than that many object names will be returned at once. Note // that the server also has a limit for how many objects to list in one // request; the lower limit wins. // // The end of the object listing is reached when an empty list is returned. // // This method offers maximal flexibility, but most users will prefer the // simpler interfaces offered by Collect() and Foreach(). func (i *ObjectIterator) NextPage(ctx context.Context, limit int) ([]*Object, error) { names, err := i.getBase().nextPage(ctx, limit) if err != nil { return nil, err } result := make([]*Object, len(names)) for idx, name := range names { result[idx] = i.Container.Object(name) } return result, nil } // The symlink_path attribute looks like "/v1/AUTH_foo/containername/obje/ctna/me". var symlinkPathRx = regexp.MustCompile(`^/v1/([^/]+)/([^/]+)/(.+)$`) // NextPageDetailed is like NextPage, but includes basic metadata. func (i *ObjectIterator) NextPageDetailed(ctx context.Context, limit int) ([]ObjectInfo, error) { b := i.getBase() var document []struct { // either all of this: SizeBytes uint64 `json:"bytes"` ContentType string `json:"content_type"` Etag string `json:"hash"` LastModifiedStr string `json:"last_modified"` Name string `json:"name"` SymlinkPath string `json:"symlink_path"` // or just this: Subdir string `json:"subdir"` } err := b.nextPageDetailed(ctx, limit, &document) if err != nil { return nil, err } if len(document) == 0 { b.setMarker("") // indicate EOF to iteratorBase return nil, nil } result := make([]ObjectInfo, len(document)) marker := "" for idx, data := range document { if data.Subdir == "" { marker = data.Name result[idx].Object = i.Container.Object(data.Name) result[idx].ContentType = data.ContentType result[idx].Etag = data.Etag result[idx].SizeBytes = data.SizeBytes result[idx].LastModified, err = time.Parse(time.RFC3339Nano, data.LastModifiedStr+"Z") if err != nil { // this error is sufficiently obscure that we don't need to expose a type for it return nil, fmt.Errorf("bad field objects[%d].last_modified: %s", idx, err.Error()) } if data.SymlinkPath != "" { match := symlinkPathRx.FindStringSubmatch(data.SymlinkPath) if match == nil { // like above return nil, fmt.Errorf("bad field objects[%d].symlink_path: %q", idx, data.SymlinkPath) } a := i.Container.a if a.Name() != match[1] { a = a.SwitchAccount(match[1]) } result[idx].SymlinkTarget = a.Container(match[2]).Object(match[3]) } } else { marker = data.Subdir result[idx].SubDirectory = data.Subdir } } b.setMarker(marker) return result, nil } // Foreach lists the object names matching this iterator and calls the // callback once for every object. Iteration is aborted when a GET request fails, // or when the callback returns a non-nil error. func (i *ObjectIterator) Foreach(ctx context.Context, callback func(*Object) error) error { for { objects, err := i.NextPage(ctx, -1) if err != nil { return err } if len(objects) == 0 { return nil // EOF } for _, o := range objects { err := callback(o) if err != nil { return err } } } } // ForeachDetailed is like Foreach, but includes basic metadata. func (i *ObjectIterator) ForeachDetailed(ctx context.Context, callback func(ObjectInfo) error) error { for { infos, err := i.NextPageDetailed(ctx, -1) if err != nil { return err } if len(infos) == 0 { return nil // EOF } for _, ci := range infos { err := callback(ci) if err != nil { return err } } } } // Collect lists all object names matching this iterator. For large sets of // objects that cannot be retrieved at once, Collect handles paging behind // the scenes. The return value is always the complete set of objects. func (i *ObjectIterator) Collect(ctx context.Context) ([]*Object, error) { var result []*Object for { objects, err := i.NextPage(ctx, -1) if err != nil { return nil, err } if len(objects) == 0 { return result, nil // EOF } result = append(result, objects...) } } // CollectDetailed is like Collect, but includes basic metadata. func (i *ObjectIterator) CollectDetailed(ctx context.Context) ([]ObjectInfo, error) { var result []ObjectInfo for { infos, err := i.NextPageDetailed(ctx, -1) if err != nil { return nil, err } if len(infos) == 0 { return result, nil // EOF } result = append(result, infos...) } }