From 9e8ed9ef479ca2084d9e34edfda0a99be34dbdb5 Mon Sep 17 00:00:00 2001 From: Stefan Majewsky Date: Thu, 8 Mar 2018 23:02:44 +0100 Subject: generalize BulkUploadError into BulkError For use in Account.BulkDelete(). --- bulk.go | 12 ++--- errors.go | 28 +++++----- object.go | 2 + tests/bulk_test.go | 135 ---------------------------------------------- tests/bulk_upload_test.go | 135 ++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 157 insertions(+), 155 deletions(-) delete mode 100644 tests/bulk_test.go create mode 100644 tests/bulk_upload_test.go diff --git a/bulk.go b/bulk.go index 448c10c..c012c42 100644 --- a/bulk.go +++ b/bulk.go @@ -56,7 +56,7 @@ const ( //exceeded in the middle of the archive extraction). // //If not nil, the error return value is *usually* an instance of -//schwift.BulkUploadError. +//BulkError. // //This operation returns (0, ErrNotSupported) if the server does not support //bulk-uploading. @@ -114,9 +114,9 @@ func (a *Account) BulkUpload(uploadPath string, format BulkUploadFormat, content return 0, err } - //parse `result` into type BulkUploadError - bulkErr := BulkUploadError{ - ArchiveError: result.ResponseBody, + //parse `result` into type BulkError + bulkErr := BulkError{ + OverallError: result.ResponseBody, } bulkErr.StatusCode, err = parseResponseStatus(result.ResponseStatus) if err != nil { @@ -141,8 +141,8 @@ func (a *Account) BulkUpload(uploadPath string, format BulkUploadFormat, content }) } - //is BulkUploadError really an error? - if len(bulkErr.ObjectErrors) == 0 && bulkErr.ArchiveError == "" && bulkErr.StatusCode >= 200 && bulkErr.StatusCode < 300 { + //is BulkError really an error? + if len(bulkErr.ObjectErrors) == 0 && bulkErr.OverallError == "" && bulkErr.StatusCode >= 200 && bulkErr.StatusCode < 300 { return result.NumberFilesCreated, nil } return result.NumberFilesCreated, bulkErr diff --git a/errors.go b/errors.go index 3ce7cd0..1eab131 100644 --- a/errors.go +++ b/errors.go @@ -66,7 +66,7 @@ func (e UnexpectedStatusCodeError) Error() string { } //BulkObjectError is the error message for a single object in a bulk operation. -//It is not generated individually, only as part of BulkUploadError and BulkDeleteError. +//It is not generated individually, only as part of BulkError. type BulkObjectError struct { ContainerName string ObjectName string @@ -81,27 +81,27 @@ func (e BulkObjectError) Error() string { ) } -//BulkUploadError is returned by Account.BulkUpload() when the archive was -//uploaded and unpacked successfully, but some (or all) files could not be -//saved in Swift. -type BulkUploadError struct { +//BulkError is returned by Account.BulkUpload() when the archive was +//uploaded and unpacked successfully, but some (or all) objects could not be +//saved in Swift; and by Account.BulkDelete() when not all requested objects +//could be deleted. +type BulkError struct { //StatusCode contains the overall HTTP status code of the operation. StatusCode int - //ArchiveError contains the error that occurred while unpacking the archive, - //or if the archive as a whole was not acceptable. If may be empty if no - //error occurred at this point. - ArchiveError string - //ObjectErrors contains errors that occurred while trying to save an - //individual file from the archive. It may be empty. + //OverallError contains the fatal error that aborted the bulk operation, or a + //summary of which recoverable errors were encountered. It may be empty. + OverallError string + //ObjectErrors contains errors that occurred while working on individual + //objects. It may be empty if no such errors occurred. ObjectErrors []BulkObjectError } //Error implements the builtin/error interface. To fit into one line, it //condenses the ObjectErrors into a count. -func (e BulkUploadError) Error() string { +func (e BulkError) Error() string { result := fmt.Sprintf("%d %s", e.StatusCode, http.StatusText(e.StatusCode)) - if e.ArchiveError != "" { - result += ": " + e.ArchiveError + if e.OverallError != "" { + result += ": " + e.OverallError } if len(e.ObjectErrors) > 0 { result += fmt.Sprintf(" (+%d object errors)", len(e.ObjectErrors)) diff --git a/object.go b/object.go index b8f5365..4af37e6 100644 --- a/object.go +++ b/object.go @@ -207,6 +207,8 @@ func (o *Object) Upload(content io.Reader, headers ObjectHeaders, opts *RequestO return nil } +//TODO add support for strings.Reader below + func tryComputeContentLength(content io.Reader, headers ObjectHeaders) { h := headers.SizeBytes() if h.Exists() { diff --git a/tests/bulk_test.go b/tests/bulk_test.go deleted file mode 100644 index b7f1e77..0000000 --- a/tests/bulk_test.go +++ /dev/null @@ -1,135 +0,0 @@ -/****************************************************************************** -* -* Copyright 2018 Stefan Majewsky -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -******************************************************************************/ - -package tests - -import ( - "archive/tar" - "bytes" - "strings" - "testing" - - "github.com/majewsky/schwift" -) - -func TestBulkUploadSuccess(t *testing.T) { - testWithContainer(t, func(c *schwift.Container) { - obj1 := c.Object("file1") - obj2 := c.Object("file2") - - archive := buildTarArchive(map[string][]byte{ - obj1.FullName(): []byte("hello"), - obj2.FullName(): []byte("world"), - }) - n, err := c.Account().BulkUpload( - "", //upload path - schwift.BulkUploadTar, - bytes.NewReader(archive), - nil, nil, - ) - expectInt(t, n, 2) - expectSuccess(t, err) - - expectObjectExistence(t, obj1, true) - expectObjectExistence(t, obj2, true) - expectObjectContent(t, obj1, []byte("hello")) - expectObjectContent(t, obj2, []byte("world")) - }) -} - -func TestBulkUploadArchiveError(t *testing.T) { - testWithContainer(t, func(c *schwift.Container) { - n, err := c.Account().BulkUpload( - c.Name(), //upload path - schwift.BulkUploadTar, - strings.NewReader("This is not the TAR archive you're looking for."), - nil, nil, - ) - expectInt(t, n, 0) - expectError(t, err, "400 Bad Request: Invalid Tar File: truncated header") - bulkErr := err.(schwift.BulkUploadError) - expectInt(t, bulkErr.StatusCode, 400) - expectString(t, bulkErr.ArchiveError, "Invalid Tar File: truncated header") - expectInt(t, len(bulkErr.ObjectErrors), 0) - }) -} - -func TestBulkUploadObjectError(t *testing.T) { - testWithContainer(t, func(c *schwift.Container) { - obj1 := c.Object(buildInvalidObjectName()) - obj2 := c.Object("file2") - expectObjectExistence(t, obj2, false) - - archive := buildTarArchive(map[string][]byte{ - obj1.Name(): []byte("hello"), - obj2.Name(): []byte("world"), - }) - n, err := c.Account().BulkUpload( - c.Name(), //upload path - schwift.BulkUploadTar, - bytes.NewReader(archive), - nil, nil, - ) - expectInt(t, n, 1) - expectError(t, err, "400 Bad Request (+1 object errors)") - bulkErr := err.(schwift.BulkUploadError) - expectInt(t, len(bulkErr.ObjectErrors), 1) - expectString(t, bulkErr.ObjectErrors[0].ContainerName, c.Name()) - expectInt(t, bulkErr.ObjectErrors[0].StatusCode, 400) - //^ We cannot match the ObjectName (or use expectError, for that matter) - //here because Swift truncates the object name to its max length. - - //even if some files cannot be processed, the other files shall be stored correctly - expectObjectExistence(t, obj2, true) - expectObjectContent(t, obj2, []byte("world")) - }) -} - -func buildTarArchive(files map[string][]byte) []byte { - var buf bytes.Buffer - w := tar.NewWriter(&buf) - for fileName, contents := range files { - err := w.WriteHeader(&tar.Header{ - Typeflag: tar.TypeReg, - Name: fileName, - Size: int64(len(contents)), - Mode: 0100644, - }) - if err != nil { - panic(err.Error()) - } - _, err = w.Write(contents) - if err != nil { - panic(err.Error()) - } - } - err := w.Close() - if err != nil { - panic(err.Error()) - } - return buf.Bytes() -} - -func buildInvalidObjectName() string { - //5000 is more than the usual max_object_name_length of 1024 - buf := make([]byte, 5000) - for idx := range buf { - buf[idx] = 'a' - } - return string(buf) -} diff --git a/tests/bulk_upload_test.go b/tests/bulk_upload_test.go new file mode 100644 index 0000000..68cb791 --- /dev/null +++ b/tests/bulk_upload_test.go @@ -0,0 +1,135 @@ +/****************************************************************************** +* +* Copyright 2018 Stefan Majewsky +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +******************************************************************************/ + +package tests + +import ( + "archive/tar" + "bytes" + "strings" + "testing" + + "github.com/majewsky/schwift" +) + +func TestBulkUploadSuccess(t *testing.T) { + testWithContainer(t, func(c *schwift.Container) { + obj1 := c.Object("file1") + obj2 := c.Object("file2") + + archive := buildTarArchive(map[string][]byte{ + obj1.FullName(): []byte("hello"), + obj2.FullName(): []byte("world"), + }) + n, err := c.Account().BulkUpload( + "", //upload path + schwift.BulkUploadTar, + bytes.NewReader(archive), + nil, nil, + ) + expectInt(t, n, 2) + expectSuccess(t, err) + + expectObjectExistence(t, obj1, true) + expectObjectExistence(t, obj2, true) + expectObjectContent(t, obj1, []byte("hello")) + expectObjectContent(t, obj2, []byte("world")) + }) +} + +func TestBulkUploadArchiveError(t *testing.T) { + testWithContainer(t, func(c *schwift.Container) { + n, err := c.Account().BulkUpload( + c.Name(), //upload path + schwift.BulkUploadTar, + strings.NewReader("This is not the TAR archive you're looking for."), + nil, nil, + ) + expectInt(t, n, 0) + expectError(t, err, "400 Bad Request: Invalid Tar File: truncated header") + bulkErr := err.(schwift.BulkError) + expectInt(t, bulkErr.StatusCode, 400) + expectString(t, bulkErr.OverallError, "Invalid Tar File: truncated header") + expectInt(t, len(bulkErr.ObjectErrors), 0) + }) +} + +func TestBulkUploadObjectError(t *testing.T) { + testWithContainer(t, func(c *schwift.Container) { + obj1 := c.Object(buildInvalidObjectName()) + obj2 := c.Object("file2") + expectObjectExistence(t, obj2, false) + + archive := buildTarArchive(map[string][]byte{ + obj1.Name(): []byte("hello"), + obj2.Name(): []byte("world"), + }) + n, err := c.Account().BulkUpload( + c.Name(), //upload path + schwift.BulkUploadTar, + bytes.NewReader(archive), + nil, nil, + ) + expectInt(t, n, 1) + expectError(t, err, "400 Bad Request (+1 object errors)") + bulkErr := err.(schwift.BulkError) + expectInt(t, len(bulkErr.ObjectErrors), 1) + expectString(t, bulkErr.ObjectErrors[0].ContainerName, c.Name()) + expectInt(t, bulkErr.ObjectErrors[0].StatusCode, 400) + //^ We cannot match the ObjectName (or use expectError, for that matter) + //here because Swift truncates the object name to its max length. + + //even if some files cannot be processed, the other files shall be stored correctly + expectObjectExistence(t, obj2, true) + expectObjectContent(t, obj2, []byte("world")) + }) +} + +func buildTarArchive(files map[string][]byte) []byte { + var buf bytes.Buffer + w := tar.NewWriter(&buf) + for fileName, contents := range files { + err := w.WriteHeader(&tar.Header{ + Typeflag: tar.TypeReg, + Name: fileName, + Size: int64(len(contents)), + Mode: 0100644, + }) + if err != nil { + panic(err.Error()) + } + _, err = w.Write(contents) + if err != nil { + panic(err.Error()) + } + } + err := w.Close() + if err != nil { + panic(err.Error()) + } + return buf.Bytes() +} + +func buildInvalidObjectName() string { + //5000 is more than the usual max_object_name_length of 1024 + buf := make([]byte, 5000) + for idx := range buf { + buf[idx] = 'a' + } + return string(buf) +} -- cgit v1.2.3