/****************************************************************************** * * Copyright 2018 Stefan Majewsky * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * ******************************************************************************/ package schwift import ( "bytes" "context" "crypto/hmac" "crypto/md5" //nolint:gosec // Etag uses md5 "crypto/sha1" //nolint:gosec // Used by swift "crypto/sha256" "encoding/hex" "errors" "fmt" "hash" "io" "net/http" "net/url" "strings" "time" ) // Object represents a Swift object. Instances are usually obtained by // traversing downwards from a container with Container.Object() or // Container.Objects(). type Object struct { c *Container name string // cache headers *ObjectHeaders // from HEAD/GET without ?symlink=get symlinkHeaders *ObjectHeaders // from HEAD/GET with ?symlink=get } // IsEqualTo returns true if both Object instances refer to the same object. func (o *Object) IsEqualTo(other *Object) bool { return other.name == o.name && other.c.IsEqualTo(o.c) } // Object returns a handle to the object with the given name within this // container. This function does not issue any HTTP requests, and therefore cannot // ensure that the object exists. Use the Exists() function to check for the // object's existence. func (c *Container) Object(name string) *Object { return &Object{c: c, name: name} } // Container returns a handle to the container this object is stored in. func (o *Object) Container() *Container { return o.c } // Name returns the object name. This does not parse the name in any way; if you // want only the basename portion of the object name, use package path from the // standard library in conjunction with this function. For example: // // obj := account.Container("docs").Object("2018-02-10/invoice.pdf") // obj.Name() //returns "2018-02-10/invoice.pdf" // path.Base(obj.Name()) //returns "invoice.pdf" func (o *Object) Name() string { return o.name } // FullName returns the container name and object name joined together with a // slash. This identifier is used by Swift in several places (large object // manifests, symlink targets, etc.) to refer to an object within an account. // For example: // // obj := account.Container("docs").Object("2018-02-10/invoice.pdf") // obj.Name() //returns "2018-02-10/invoice.pdf" // obj.FullName() //returns "docs/2018-02-10/invoice.pdf" func (o *Object) FullName() string { return o.c.name + "/" + o.name } // Exists checks if this object exists, potentially by issuing a HEAD request // if no Headers() have been cached yet. func (o *Object) Exists(ctx context.Context) (bool, error) { _, err := o.Headers(ctx) if Is(err, http.StatusNotFound) { return false, nil } else if err != nil { return false, err } return true, nil } // Headers returns the ObjectHeaders for this object. If the ObjectHeaders // has not been cached yet, a HEAD request is issued on the object. // // For symlinks, this operation returns the metadata for the target object. Use // Object.SymlinkHeaders() to obtain the metadata for the symlink instead. // // This operation fails with http.StatusNotFound if the object does not exist. // // WARNING: This method is not thread-safe. Calling it concurrently on the same // object results in undefined behavior. func (o *Object) Headers(ctx context.Context) (ObjectHeaders, error) { if o.headers != nil { return *o.headers, nil } hdr, err := o.fetchHeaders(ctx, nil) if err != nil { return ObjectHeaders{}, err } o.headers = hdr return *hdr, nil } func (o *Object) fetchHeaders(ctx context.Context, opts *RequestOptions) (*ObjectHeaders, error) { resp, err := Request{ Method: "HEAD", ContainerName: o.c.name, ObjectName: o.name, Options: opts, // since Openstack LOVES to be inconsistent with everything (incl. itself), // this returns 200 instead of 204 ExpectStatusCodes: []int{http.StatusOK}, DrainResponseBody: true, }.Do(ctx, o.c.a.backend) if err != nil { return nil, err } defer resp.Body.Close() headers := ObjectHeaders{headersFromHTTP(resp.Header)} return &headers, headers.Validate() } // Update updates the object's headers using a POST request. To add URL // parameters, pass a non-nil *RequestOptions. // // This operation fails with http.StatusNotFound if the object does not exist. // // A successful POST request implies Invalidate() since it may change metadata. func (o *Object) Update(ctx context.Context, headers ObjectHeaders, opts *RequestOptions) error { resp, err := Request{ Method: "POST", ContainerName: o.c.name, ObjectName: o.name, Options: cloneRequestOptions(opts, headers.Headers), ExpectStatusCodes: []int{http.StatusAccepted}, }.Do(ctx, o.c.a.backend) if err == nil { o.Invalidate() resp.Body.Close() } return err } // UploadOptions invokes advanced behavior in the Object.Upload() method. type UploadOptions struct { // When overwriting a large object, delete its segments. This will cause // Upload() to call into BulkDelete(), so a BulkError may be returned. DeleteSegments bool } // Upload creates the object using a PUT request. // // If you do not have an io.Reader, but you have a []byte or string instance // containing the object, wrap it in a *bytes.Reader instance like so: // // var buffer []byte // o.Upload(bytes.NewReader(buffer), opts) // // //or... // var buffer string // o.Upload(bytes.NewReader([]byte(buffer)), opts) // // If you have neither an io.Reader nor a []byte or string, but you have a // function that generates the object's content into an io.Writer, use // UploadFromWriter instead. // // If the object is very large and you want to upload it in segments, use // LargeObject.Append() instead. See documentation on type LargeObject for // details. // // If content is a *bytes.Reader or a *bytes.Buffer instance, the Content-Length // and Etag request headers will be computed automatically. Otherwise, it is // highly recommended that the caller set these headers (if possible) to allow // the server to check the integrity of the uploaded file. // // If Etag and/or Content-Length is supplied and the content does not match // these parameters, http.StatusUnprocessableEntity is returned. If Etag is not // supplied and cannot be computed in advance, Upload() will compute the Etag as // data is read from the io.Reader, and compare the result to the Etag returned // by Swift, returning ErrChecksumMismatch in case of mismatch. The object will // have been uploaded at that point, so you will usually want to Delete() it. // // This function can be used regardless of whether the object exists or not. // // A successful PUT request implies Invalidate() since it may change metadata. func (o *Object) Upload(ctx context.Context, content io.Reader, opts *UploadOptions, ropts *RequestOptions) error { if opts == nil { opts = &UploadOptions{} } ropts = cloneRequestOptions(ropts, nil) hdr := ObjectHeaders{ropts.Headers} if !hdr.SizeBytes().Exists() { value := tryComputeContentLength(content) if value != nil { hdr.SizeBytes().Set(*value) } } // do not attempt to add the Etag header when we're writing a large object // manifest; the header refers to the content, but we would be computing the // manifest's hash instead isManifestUpload := ropts.Values.Get("multipart-manifest") == "put" || hdr.IsDynamicLargeObject() var hasher hash.Hash if !isManifestUpload { err := tryComputeEtag(content, hdr) if err != nil { return err } // could not compute Etag in advance -> need to check on the fly if !hdr.Etag().Exists() { hasher = md5.New() //nolint:gosec // Etag uses md5 if content != nil { content = io.TeeReader(content, hasher) } } } var lo *LargeObject if opts.DeleteSegments { // enumerate segments in large object before overwriting it, but only delete // the segments after successfully uploading the new object to decrease the // chance of an inconsistent state following an upload error var err error lo, err = o.AsLargeObject(ctx) switch { case err == nil: // okay, delete segments at the end case errors.Is(err, ErrNotLarge): // okay, do not try to delete segments lo = nil default: // unexpected error return err } } resp, err := Request{ Method: "PUT", ContainerName: o.c.name, ObjectName: o.name, Options: ropts, Body: content, ExpectStatusCodes: []int{201}, DrainResponseBody: true, }.Do(ctx, o.c.a.backend) if err != nil { return err } o.Invalidate() defer resp.Body.Close() if hasher != nil { expectedEtag := hex.EncodeToString(hasher.Sum(nil)) if expectedEtag != resp.Header.Get("Etag") { return ErrChecksumMismatch } } if opts.DeleteSegments && lo != nil { _, _, err := lo.object.c.a.BulkDelete(ctx, lo.SegmentObjects(), nil, nil) if err != nil { return err } } return nil } type readerWithLen interface { // Returns the number of bytes in the unread portion of the buffer. // Implemented by bytes.Reader, bytes.Buffer and strings.Reader. Len() int } func tryComputeContentLength(content io.Reader) *uint64 { if content == nil { val := uint64(0) return &val } else if r, ok := content.(readerWithLen); ok { val := uint64(r.Len()) return &val } return nil } //nolint:gosec // Etag uses md5 func tryComputeEtag(content io.Reader, headers ObjectHeaders) error { h := headers.Etag() if h.Exists() { return nil } switch r := content.(type) { case nil: sum := md5.Sum(nil) h.Set(hex.EncodeToString(sum[:])) case *bytes.Buffer: // bytes.Buffer has a method that returns the unread portion of the buffer, // so this one is easy sum := md5.Sum(r.Bytes()) h.Set(hex.EncodeToString(sum[:])) case io.ReadSeeker: // bytes.Reader does not have such a method, but it is an io.Seeker, so we // can read the entire thing and then seek back to where we started md5Hash := md5.New() n, err := io.Copy(md5Hash, r) if err != nil { return err } _, err = r.Seek(-n, io.SeekCurrent) if err != nil { return err } h.Set(hex.EncodeToString(md5Hash.Sum(nil))) } return nil } // UploadFromWriter is a variant of Upload that can be used when the object's // content is generated by some function or package that takes an io.Writer // instead of supplying an io.Reader. For example: // // func greeting(target io.Writer, name string) error { // _, err := fmt.Fprintf(target, "Hello %s!\n", name) // return err // } // // obj := container.Object("greeting-for-susan-and-jeffrey") // err := obj.UploadFromWriter(nil, func(w io.Writer) error { // err := greeting(w, "Susan") // if err == nil { // err = greeting(w, "Jeffrey") // } // return err // }) // // If you do not need an io.Writer, always use Upload instead. func (o *Object) UploadFromWriter(ctx context.Context, opts *UploadOptions, ropts *RequestOptions, callback func(io.Writer) error) error { reader, writer := io.Pipe() errChan := make(chan error) go func() { err := o.Upload(ctx, reader, opts, ropts) reader.CloseWithError(err) // stop the writer if it is still writing errChan <- err }() writer.CloseWithError(callback(writer)) // stop the reader if it is still reading return <-errChan } // DeleteOptions invokes advanced behavior in the Object.Delete() method. type DeleteOptions struct { // When deleting a large object, also delete its segments. This will cause // Delete() to call into BulkDelete(), so a BulkError may be returned. DeleteSegments bool } // Delete deletes the object using a DELETE request. To add URL parameters, // pass a non-nil *RequestOptions. // // This operation fails with http.StatusNotFound if the object does not exist. // // A successful DELETE request implies Invalidate(). func (o *Object) Delete(ctx context.Context, opts *DeleteOptions, ropts *RequestOptions) error { if opts == nil { opts = &DeleteOptions{} } if opts.DeleteSegments { exists, err := o.Exists(ctx) if err != nil { return err } if exists { lo, err := o.AsLargeObject(ctx) switch { case err == nil: // is large object - delete segments and the object itself in one step _, _, err := o.c.a.BulkDelete(ctx, append(lo.SegmentObjects(), o), nil, nil) o.Invalidate() return err case errors.Is(err, ErrNotLarge): // not a large object - use regular DELETE request default: // unexpected error return err } } } resp, err := Request{ Method: "DELETE", ContainerName: o.c.name, ObjectName: o.name, Options: ropts, ExpectStatusCodes: []int{http.StatusNoContent}, }.Do(ctx, o.c.a.backend) if err == nil { o.Invalidate() resp.Body.Close() } return err } // Invalidate clears the internal cache of this Object instance. The next call // to Headers() on this instance will issue a HEAD request on the object. // // WARNING: This method is not thread-safe. Calling it concurrently on the same // object results in undefined behavior. func (o *Object) Invalidate() { o.headers = nil o.symlinkHeaders = nil } // Download retrieves the object's contents using a GET request. This returns a // helper object which allows you to select whether you want an io.ReadCloser // for reading the object contents progressively, or whether you want the object // contents collected into a byte slice or string. // // reader, err := object.Download(nil).AsReadCloser() // // buf, err := object.Download(nil).AsByteSlice() // // str, err := object.Download(nil).AsString() // // See documentation on type DownloadedObject for details. // // WARNING: This method is not thread-safe. Calling it concurrently on the same // object results in undefined behavior. func (o *Object) Download(ctx context.Context, opts *RequestOptions) DownloadedObject { resp, err := Request{ Method: "GET", ContainerName: o.c.name, ObjectName: o.name, Options: opts, ExpectStatusCodes: []int{http.StatusOK}, }.Do(ctx, o.c.a.backend) //nolint:bodyclose // body is returned and must be closed by the user var body io.ReadCloser if err == nil { newHeaders := ObjectHeaders{headersFromHTTP(resp.Header)} err = newHeaders.Validate() if err == nil { if opts != nil && opts.Values != nil && opts.Values.Get("symlink") == "get" { o.symlinkHeaders = &newHeaders } else { o.headers = &newHeaders } } body = resp.Body } return DownloadedObject{body, err} } // CopyOptions invokes advanced behavior in the Object.Copy() method. type CopyOptions struct { // Copy only the object's content, not its metadata. New metadata can always // be supplied in the RequestOptions argument of Object.CopyTo(). FreshMetadata bool // When the source is a symlink, copy the symlink instead of the target object. ShallowCopySymlinks bool } // CopyTo copies the object on the server side using a COPY request. // // A successful COPY implies target.Invalidate() since it may change the // target's metadata. func (o *Object) CopyTo(ctx context.Context, target *Object, opts *CopyOptions, ropts *RequestOptions) error { ropts = cloneRequestOptions(ropts, nil) ropts.Headers.Set("Destination", target.FullName()) if o.c.a.name != target.c.a.name { ropts.Headers.Set("Destination-Account", target.c.a.name) } if opts != nil { if opts.FreshMetadata { ropts.Headers.Set("X-Fresh-Metadata", "true") } if opts.ShallowCopySymlinks { ropts.Values.Set("symlink", "get") } } resp, err := Request{ Method: "COPY", ContainerName: o.c.name, ObjectName: o.name, Options: ropts, ExpectStatusCodes: []int{http.StatusCreated}, DrainResponseBody: true, }.Do(ctx, o.c.a.backend) if err == nil { target.Invalidate() resp.Body.Close() } return err } // SymlinkOptions invokes advanced behavior in the Object.SymlinkTo() method. type SymlinkOptions struct { // When overwriting a large object, delete its segments. This will cause // SymlinkTo() to call into BulkDelete(), so a BulkError may be returned. DeleteSegments bool } // SymlinkTo creates the object as a symbolic link to another object using a PUT // request. Like Object.Upload(), this method works regardless of whether the // object already exists or not. Existing object contents will be overwritten by // this operation. // // A successful PUT request implies Invalidate() since it may change metadata. func (o *Object) SymlinkTo(ctx context.Context, target *Object, opts *SymlinkOptions, ropts *RequestOptions) error { ropts = cloneRequestOptions(ropts, nil) ropts.Headers.Set("X-Symlink-Target", target.FullName()) if !target.c.a.IsEqualTo(o.c.a) { ropts.Headers.Set("X-Symlink-Target-Account", target.c.a.Name()) } if ropts.Headers.Get("Content-Type") == "" { // recommended Content-Type for symlinks as per // ropts.Headers.Set("Content-Type", "application/symlink") } var uopts *UploadOptions if opts != nil { uopts = &UploadOptions{ DeleteSegments: opts.DeleteSegments, } } return o.Upload(ctx, nil, uopts, ropts) } // SymlinkHeaders is similar to Headers, but if the object is a symlink, it // returns the metadata of the symlink rather than the metadata of the target. // It also returns a reference to the target object. // // If this object is not a symlink, Object.SymlinkHeaders() returns the same // ObjectHeaders as Object.Headers(), and a nil target object. // // In a nutshell, if Object.Headers() is like os.Stat(), then // Object.SymlinkHeaders() is like os.Lstat(). // // If you do not know whether a given object is a symlink or not, it's a good // idea to call Object.SymlinkHeaders() first: If the object turns out not to be // a symlink, the cache for Object.Headers() has already been populated. // // This operation fails with http.StatusNotFound if the object does not exist. // // WARNING: This method is not thread-safe. Calling it concurrently on the same // object results in undefined behavior. func (o *Object) SymlinkHeaders(ctx context.Context) (headers ObjectHeaders, target *Object, err error) { if o.symlinkHeaders == nil { o.symlinkHeaders, err = o.fetchHeaders(ctx, &RequestOptions{ Values: url.Values{"symlink": []string{"get"}}, }) if err != nil { return ObjectHeaders{}, nil, err } } // is this a symlink? targetFullName := o.symlinkHeaders.Get("X-Symlink-Target") if targetFullName == "" { // not a symlink - the o.symlinkHeaders are just the regular headers o.headers = o.symlinkHeaders return *o.headers, nil, nil } fields := strings.SplitN(targetFullName, "/", 2) if len(fields) < 2 { return ObjectHeaders{}, nil, MalformedHeaderError{ Key: "X-Symlink-Target", ParseError: fmt.Errorf("expected \"container/object\", got \"%s\"", targetFullName), } } // cross-account symlink? accountName := o.symlinkHeaders.Get("X-Symlink-Target-Account") targetAccount := o.c.a if accountName != "" && accountName != targetAccount.Name() { targetAccount = targetAccount.SwitchAccount(accountName) } target = targetAccount.Container(fields[0]).Object(fields[1]) return *o.symlinkHeaders, target, nil } // URL returns the canonical URL for the object on the server. This is // particularly useful when the ReadACL on the account or container is set to // allow anonymous read access. func (o *Object) URL() (string, error) { return Request{ ContainerName: o.c.name, ObjectName: o.name, }.URL(o.c.a.backend, nil) } // Returns true if string is contained in slice func contains(s []string, e string) bool { for _, a := range s { if a == e { return true } } return false } // TempURL is like Object.URL, but includes a token with a limited lifetime (as // specified by the `expires` argument) that permits anonymous access to this // object using the given HTTP method. This works only when the tempurl // middleware is set up on the server, and if the given `key` matches one of the // tempurl keys for this object's container or account. // // For example, if the ReadACL both on the account and container do not permit // anonymous read access (which is the default behavior): // // var o *schwift.Object // ... // resp, err := http.Get(o.URL()) // //After this, resp.StatusCode == 401 (Unauthorized) // //because anonymous access is forbidden. // // //But if the container or account has a tempurl key... // key := "supersecretkey" // hdr := NewContainerHeaders() // hdr.TempURLKey().Set(key) // c := o.Container() // err := c.Update(ctx, hdr, nil) // // //...we can use it to generate temporary URLs. // url := o.TempURL(ctx, key, "GET", time.Now().Add(10 * time.Minute)) // resp, err := http.Get(url) // //This time, resp.StatusCode == 200 because the URL includes a token. func (o *Object) TempURL(ctx context.Context, key, method string, expires time.Time) (string, error) { urlStr, err := o.URL() if err != nil { return "", err } u, err := url.Parse(urlStr) if err != nil { return "", err } capabilities, err := o.c.a.Capabilities(ctx) if err != nil { return "", err } allowedDigest := capabilities.TempURL.AllowedDigests var mac hash.Hash switch { case contains(allowedDigest, "sha256"): mac = hmac.New(sha256.New, []byte(key)) case contains(allowedDigest, "sha1"): mac = hmac.New(sha1.New, []byte(key)) default: return "", fmt.Errorf("schwift supports sha1 and sha256 digests but the Swift server only supports: %s", strings.Join(allowedDigest, ", ")) } payload := fmt.Sprintf("%s\n%d\n%s", method, expires.Unix(), u.Path) mac.Write([]byte(payload)) signature := hex.EncodeToString(mac.Sum(nil)) u.RawQuery = fmt.Sprintf("temp_url_sig=%s&temp_url_expires=%d", signature, expires.Unix()) return u.String(), nil }