From f9749638e3393f471d7e28362795689bf37cc023 Mon Sep 17 00:00:00 2001 From: Stefan Majewsky Date: Mon, 30 Apr 2018 14:14:56 +0200 Subject: revamp the LargeObject API I thought about this some more, and I believe the Writer-based approach in the previous version of the LargeObject API does not scale: It makes it very hard to write code that uploads segments without resorting to a buffer the same size as the segments. I don't want gigabyte-scale buffers filling up my RAM, so this commit switches to a different API based on Readers. LargeObject.Append() now behaves very similar to Object.Upload(), which I find quite nice. --- tests/bulk_delete_test.go | 2 +- tests/field_test.go | 2 +- tests/largeobject_test.go | 174 +++++++++++++++++------------------------- tests/object_iterator_test.go | 2 +- tests/object_test.go | 22 +++--- 5 files changed, 83 insertions(+), 119 deletions(-) (limited to 'tests') diff --git a/tests/bulk_delete_test.go b/tests/bulk_delete_test.go index c498419..dfe8468 100644 --- a/tests/bulk_delete_test.go +++ b/tests/bulk_delete_test.go @@ -77,7 +77,7 @@ func createTestObjects(c *schwift.Container) ([]*schwift.Object, error) { var objs []*schwift.Object for idx := 1; idx <= 5; idx++ { obj := c.Object(fmt.Sprintf("object%d", idx)) - err := obj.Upload(strings.NewReader("example"), nil) + err := obj.Upload(strings.NewReader("example"), nil, nil) if err != nil { return nil, err } diff --git a/tests/field_test.go b/tests/field_test.go index 8166f2d..c8dcf7f 100644 --- a/tests/field_test.go +++ b/tests/field_test.go @@ -88,7 +88,7 @@ func TestFieldTimestamp(t *testing.T) { func TestFieldHTTPTimestamp(t *testing.T) { testWithContainer(t, func(c *schwift.Container) { obj := c.Object("test") - err := obj.Upload(nil, nil) + err := obj.Upload(nil, nil, nil) if !expectSuccess(t, err) { return } diff --git a/tests/largeobject_test.go b/tests/largeobject_test.go index 758083d..3e41cb3 100644 --- a/tests/largeobject_test.go +++ b/tests/largeobject_test.go @@ -38,15 +38,7 @@ func TestLargeObjectsBasic(t *testing.T) { obj := c.Object(strategyStr + "-largeobject") lo, err := obj.AsLargeObject() - expectSuccess(t, err) - - //Open fails when SegmentContainer is not set - _, err = lo.Open(schwift.OpenTruncate) - expectError(t, err, schwift.ErrNoContainerName.Error()) - - lo.SegmentContainer = c - lo.SegmentPrefix = strategyStr + "-segments/" - lo.Strategy = strategy + expectError(t, err, schwift.ErrNotLarge.Error()) segment1 := getRandomSegmentContent(128) segment2 := getRandomSegmentContent(128) @@ -54,16 +46,14 @@ func TestLargeObjectsBasic(t *testing.T) { segment4 := getRandomSegmentContent(128) //basic write example - w, err := lo.Open(schwift.OpenTruncate) - expectSuccess(t, err) - if w == nil { - t.FailNow() - } - _, err = w.Write([]byte(segment1)) + lo, err = obj.AsNewLargeObject(schwift.SegmentingOptions{ + SegmentContainer: c, + SegmentPrefix: strategyStr + "-segments/", + Strategy: strategy, + }, nil) expectSuccess(t, err) - _, err = w.Write([]byte(segment2)) - expectSuccess(t, err) - expectSuccess(t, w.Close()) + expectSuccess(t, lo.Append(bytes.NewReader([]byte(segment1+segment2)), 128)) + expectSuccess(t, lo.WriteManifest(nil)) expectObjectContent(t, obj, []byte(segment1+segment2)) expectLargeObject(t, obj, []schwift.SegmentInfo{ @@ -84,16 +74,8 @@ func TestLargeObjectsBasic(t *testing.T) { expectSuccess(t, err) expectLargeObjectSetup(t, lo, strategy, fmt.Sprintf("%s/%s-segments/", c.Name(), strategyStr)) - w, err = lo.Open(schwift.OpenAppend) - expectSuccess(t, err) - if w == nil { - t.FailNow() - } - _, err = w.Write([]byte(segment3)) - expectSuccess(t, err) - _, err = w.Write([]byte(segment4)) - expectSuccess(t, err) - expectSuccess(t, w.Close()) + expectSuccess(t, lo.Append(bytes.NewReader([]byte(segment3+segment4)), 128)) + expectSuccess(t, lo.WriteManifest(nil)) expectObjectContent(t, obj, []byte(segment1+segment2+segment3+segment4)) expectLargeObject(t, obj, []schwift.SegmentInfo{ @@ -122,26 +104,22 @@ func TestLargeObjectsBasic(t *testing.T) { //basic truncate example lo, err = obj.AsLargeObject() expectSuccess(t, err) + err = lo.Truncate(&schwift.TruncateOptions{ + DeleteSegments: true, + }) + expectSuccess(t, err) expectLargeObjectSetup(t, lo, strategy, fmt.Sprintf("%s/%s-segments/", c.Name(), strategyStr)) - w, err = lo.Open(schwift.OpenTruncate) - expectSuccess(t, err) - if w == nil { - t.FailNow() - } //verify that segments were deleted iter := c.Objects() - iter.Prefix = lo.SegmentPrefix + iter.Prefix = lo.SegmentPrefix() names, err := iter.Collect() expectSuccess(t, err) expectObjectNames(t, names) - _, err = w.Write([]byte(segment3)) - expectSuccess(t, err) - _, err = w.Write([]byte(segment4)) - expectSuccess(t, err) - expectSuccess(t, w.Close()) + expectSuccess(t, lo.Append(bytes.NewReader([]byte(segment3+segment4)), 128)) + expectSuccess(t, lo.WriteManifest(nil)) expectObjectContent(t, obj, []byte(segment3+segment4)) expectLargeObject(t, obj, []schwift.SegmentInfo{ @@ -164,7 +142,7 @@ func TestLargeObjectsBasic(t *testing.T) { func TestOpenRegularObjectAsLargeObject(t *testing.T) { testWithContainer(t, func(c *schwift.Container) { o := c.Object("foo") - expectSuccess(t, o.Upload(bytes.NewReader(objectExampleContent), nil)) + expectSuccess(t, o.Upload(bytes.NewReader(objectExampleContent), nil, nil)) _, err := o.AsLargeObject() expectError(t, err, schwift.ErrNotLarge.Error()) }) @@ -173,29 +151,21 @@ func TestOpenRegularObjectAsLargeObject(t *testing.T) { func TestSLOWithDataSegment(t *testing.T) { testWithContainer(t, func(c *schwift.Container) { o := c.Object("foo") - lo, err := o.AsLargeObject() + lo, err := o.AsNewLargeObject(schwift.SegmentingOptions{ + SegmentContainer: c, + SegmentPrefix: "segments/", + Strategy: schwift.StaticLargeObject, + }, nil) expectSuccess(t, err) - lo.SegmentContainer = c - lo.SegmentPrefix = "segments/" - lo.Strategy = schwift.StaticLargeObject segment1 := getRandomSegmentContent(128) - segment2 := getRandomSegmentContent(128) - w, err := lo.Open(schwift.OpenTruncate) - expectSuccess(t, err) - if w == nil { - t.FailNow() - } - _, err = w.Write([]byte(segment1)) - expectSuccess(t, err) - dataSegment := schwift.SegmentInfo{Data: []byte("---")} - err = lo.AddSegment(dataSegment) - expectSuccess(t, err) + segment2 := getRandomSegmentContent(128) - _, err = w.Write([]byte(segment2)) - expectSuccess(t, err) - expectSuccess(t, w.Close()) + expectSuccess(t, lo.Append(bytes.NewReader([]byte(segment1)), 0)) + expectSuccess(t, lo.AddSegment(dataSegment)) + expectSuccess(t, lo.Append(bytes.NewReader([]byte(segment2)), 0)) + expectSuccess(t, lo.WriteManifest(nil)) expectObjectContent(t, o, []byte(segment1+string(dataSegment.Data)+segment2)) expectLargeObject(t, o, []schwift.SegmentInfo{ @@ -218,14 +188,15 @@ func TestSLOWithRangeSegments(t *testing.T) { testWithContainer(t, func(c *schwift.Container) { segmentStr := "XX" segmentObj := c.Object("segment") - expectSuccess(t, segmentObj.Upload(bytes.NewReader([]byte(segmentStr)), nil)) + expectSuccess(t, segmentObj.Upload(bytes.NewReader([]byte(segmentStr)), nil, nil)) o := c.Object("largeobject") - lo, err := o.AsLargeObject() + lo, err := o.AsNewLargeObject(schwift.SegmentingOptions{ + SegmentContainer: c, + SegmentPrefix: "segments/", + Strategy: schwift.StaticLargeObject, + }, nil) expectSuccess(t, err) - lo.SegmentContainer = c - lo.SegmentPrefix = "segments/" - lo.Strategy = schwift.StaticLargeObject //the large object is composed out of three ranges such that the "X" are precisely cut out of segmentStr expectSuccess(t, lo.AddSegment(schwift.SegmentInfo{ @@ -277,26 +248,23 @@ func TestSLOGuessSegmentPrefix(t *testing.T) { obj := c.Object("largeobject") //setup phase: create an SLO - lo, err := obj.AsLargeObject() - expectSuccess(t, err) - lo.SegmentContainer = c - lo.SegmentPrefix = "foo/bar/baz/" - w, err := lo.Open(schwift.OpenTruncate) + lo, err := obj.AsNewLargeObject(schwift.SegmentingOptions{ + SegmentContainer: c, + SegmentPrefix: "foo/bar/baz/", + }, nil) expectSuccess(t, err) segment1 := getRandomSegmentContent(128) segment2 := getRandomSegmentContent(128) - _, err = w.Write([]byte(segment1)) - expectSuccess(t, err) - _, err = w.Write([]byte(segment2)) - expectSuccess(t, err) - expectSuccess(t, w.Close()) + expectSuccess(t, lo.Append(bytes.NewReader([]byte(segment1)), 0)) + expectSuccess(t, lo.Append(bytes.NewReader([]byte(segment2)), 0)) + expectSuccess(t, lo.WriteManifest(nil)) //now create a fresh SLO and check if it infers the correct SegmentPrefix lo, err = obj.AsLargeObject() expectSuccess(t, err) - expectString(t, lo.SegmentContainer.Name(), c.Name()) - expectString(t, lo.SegmentPrefix, "foo/bar/baz/") + expectString(t, lo.SegmentContainer().Name(), c.Name()) + expectString(t, lo.SegmentPrefix(), "foo/bar/baz/") }) } @@ -305,27 +273,25 @@ func TestDeleteLargeObjectAndKeepSegments(t *testing.T) { testWithContainer(t, func(c *schwift.Container) { obj := c.Object("largeobject") - //setup phase: create an SLO - lo, err := obj.AsLargeObject() - expectSuccess(t, err) - lo.SegmentContainer = c - lo.SegmentPrefix = "foo/bar/baz/" - w, err := lo.Open(schwift.OpenTruncate) + //setup phase: create a large object + lo, err := obj.AsNewLargeObject(schwift.SegmentingOptions{ + SegmentContainer: c, + SegmentPrefix: "foo/bar/baz/", + Strategy: strategy, + }, nil) expectSuccess(t, err) segment1 := getRandomSegmentContent(128) segment2 := getRandomSegmentContent(128) - _, err = w.Write([]byte(segment1)) - expectSuccess(t, err) - _, err = w.Write([]byte(segment2)) - expectSuccess(t, err) - expectSuccess(t, w.Close()) + expectSuccess(t, lo.Append(bytes.NewReader([]byte(segment1)), 0)) + expectSuccess(t, lo.Append(bytes.NewReader([]byte(segment2)), 0)) + expectSuccess(t, lo.WriteManifest(nil)) //test deletion that keeps segments expectSuccess(t, obj.Delete(nil, nil)) iter := c.Objects() - iter.Prefix = lo.SegmentPrefix + iter.Prefix = lo.SegmentPrefix() names, err := iter.Collect() expectSuccess(t, err) expectObjectNames(t, names, @@ -340,27 +306,25 @@ func TestDeleteLargeObjectIncludingSegments(t *testing.T) { testWithContainer(t, func(c *schwift.Container) { obj := c.Object("largeobject") - //setup phase: create an SLO - lo, err := obj.AsLargeObject() - expectSuccess(t, err) - lo.SegmentContainer = c - lo.SegmentPrefix = "foo/bar/baz/" - w, err := lo.Open(schwift.OpenTruncate) + //setup phase: create a large object + lo, err := obj.AsNewLargeObject(schwift.SegmentingOptions{ + SegmentContainer: c, + SegmentPrefix: "foo/bar/baz/", + Strategy: strategy, + }, nil) expectSuccess(t, err) segment1 := getRandomSegmentContent(128) segment2 := getRandomSegmentContent(128) - _, err = w.Write([]byte(segment1)) - expectSuccess(t, err) - _, err = w.Write([]byte(segment2)) - expectSuccess(t, err) - expectSuccess(t, w.Close()) + expectSuccess(t, lo.Append(bytes.NewReader([]byte(segment1)), 0)) + expectSuccess(t, lo.Append(bytes.NewReader([]byte(segment2)), 0)) + expectSuccess(t, lo.WriteManifest(nil)) //test deletion that keeps segments expectSuccess(t, obj.Delete(&schwift.DeleteOptions{DeleteSegments: true}, nil)) iter := c.Objects() - iter.Prefix = lo.SegmentPrefix + iter.Prefix = lo.SegmentPrefix() names, err := iter.Collect() expectSuccess(t, err) expectObjectNames(t, names) @@ -415,19 +379,19 @@ func expectLargeObject(t *testing.T, obj *schwift.Object, expected []schwift.Seg } func expectLargeObjectSetup(t *testing.T, lo *schwift.LargeObject, strategy schwift.LargeObjectStrategy, segmentFullPrefix string) { - if strategy != lo.Strategy { + if strategy != lo.Strategy() { t.Errorf("expected %s to use LargeObjectStrategy %d, got %d", - lo.Object.FullName(), strategy, lo.Strategy) + lo.Object().FullName(), strategy, lo.Strategy()) } - if lo.SegmentContainer == nil { + if lo.SegmentContainer() == nil { t.Errorf("expected %s to use segment container+prefix %q, got no container", - lo.Object.FullName(), segmentFullPrefix) + lo.Object().FullName(), segmentFullPrefix) } else { - fullPrefix := lo.SegmentContainer.Name() + "/" + lo.SegmentPrefix + fullPrefix := lo.SegmentContainer().Name() + "/" + lo.SegmentPrefix() if fullPrefix != segmentFullPrefix { t.Errorf("expected %s to use segment container+prefix %q, got %q", - lo.Object.FullName(), segmentFullPrefix, fullPrefix) + lo.Object().FullName(), segmentFullPrefix, fullPrefix) } } } diff --git a/tests/object_iterator_test.go b/tests/object_iterator_test.go index 009189d..9b822cd 100644 --- a/tests/object_iterator_test.go +++ b/tests/object_iterator_test.go @@ -39,7 +39,7 @@ func TestObjectIterator(t *testing.T) { for idx := 1; idx <= 4; idx++ { hdr := schwift.NewObjectHeaders() hdr.ContentType().Set("application/json") - err := c.Object(oname(idx)).Upload(bytes.NewReader(objectExampleContent), hdr.ToOpts()) + err := c.Object(oname(idx)).Upload(bytes.NewReader(objectExampleContent), nil, hdr.ToOpts()) expectSuccess(t, err) } diff --git a/tests/object_test.go b/tests/object_test.go index 6291527..251db9f 100644 --- a/tests/object_test.go +++ b/tests/object_test.go @@ -51,7 +51,7 @@ func TestObjectLifecycle(t *testing.T) { err = o.Delete(nil, nil) expectError(t, err, "expected 204 response, got 404 instead:

Not Found

The resource could not be found.

") - err = o.Upload(bytes.NewReader([]byte("test")), nil) + err = o.Upload(bytes.NewReader([]byte("test")), nil, nil) expectSuccess(t, err) expectObjectExistence(t, o, true) @@ -66,31 +66,31 @@ func TestObjectUpload(t *testing.T) { //test upload with bytes.Reader obj := c.Object("upload1") - err := obj.Upload(bytes.NewReader(objectExampleContent), nil) + err := obj.Upload(bytes.NewReader(objectExampleContent), nil, nil) expectSuccess(t, err) expectObjectContent(t, obj, objectExampleContent) //test upload with bytes.Buffer obj = c.Object("upload2") - err = obj.Upload(bytes.NewBuffer(objectExampleContent), nil) + err = obj.Upload(bytes.NewBuffer(objectExampleContent), nil, nil) expectSuccess(t, err) expectObjectContent(t, obj, objectExampleContent) //test upload with strings.Reader obj = c.Object("upload3") - err = obj.Upload(strings.NewReader(string(objectExampleContent)), nil) + err = obj.Upload(strings.NewReader(string(objectExampleContent)), nil, nil) expectSuccess(t, err) expectObjectContent(t, obj, objectExampleContent) //test upload with opaque io.Reader obj = c.Object("upload4") - err = obj.Upload(opaqueReader{bytes.NewReader(objectExampleContent)}, nil) + err = obj.Upload(opaqueReader{bytes.NewReader(objectExampleContent)}, nil, nil) expectSuccess(t, err) expectObjectContent(t, obj, objectExampleContent) //test upload with io.Writer obj = c.Object("upload5") - err = obj.UploadWithWriter(nil, func(w io.Writer) error { + err = obj.UploadWithWriter(nil, nil, func(w io.Writer) error { _, err := w.Write(objectExampleContent) return err }) @@ -99,13 +99,13 @@ func TestObjectUpload(t *testing.T) { //test upload with empty reader (should create zero-byte-sized object) obj = c.Object("upload6") - err = obj.Upload(eofReader{}, nil) + err = obj.Upload(eofReader{}, nil, nil) expectSuccess(t, err) expectObjectContent(t, obj, nil) //test upload without reader (should create zero-byte-sized object) obj = c.Object("upload7") - err = obj.Upload(nil, nil) + err = obj.Upload(nil, nil, nil) expectSuccess(t, err) expectObjectContent(t, obj, nil) }) @@ -129,7 +129,7 @@ func TestObjectDownload(t *testing.T) { testWithContainer(t, func(c *schwift.Container) { //upload example object obj := c.Object("example") - err := obj.Upload(bytes.NewReader(objectExampleContent), nil) + err := obj.Upload(bytes.NewReader(objectExampleContent), nil, nil) expectSuccess(t, err) //test download as string @@ -170,7 +170,7 @@ func TestObjectUpdate(t *testing.T) { expectError(t, err, "expected 202 response, got 404 instead:

Not Found

The resource could not be found.

") //create object - err = obj.Upload(nil, nil) + err = obj.Upload(nil, nil, nil) expectSuccess(t, err) hdr, err := obj.Headers() @@ -190,7 +190,7 @@ func TestObjectUpdate(t *testing.T) { func TestObjectCopyMove(t *testing.T) { testWithContainer(t, func(c *schwift.Container) { obj1 := c.Object("location1") - err := obj1.Upload(bytes.NewReader(objectExampleContent), nil) + err := obj1.Upload(bytes.NewReader(objectExampleContent), nil, nil) expectSuccess(t, err) expectObjectExistence(t, obj1, true) -- cgit v1.2.3