From 0df55a731aa3330f82d22b010a7a2a4d66521972 Mon Sep 17 00:00:00 2001 From: Stefan Majewsky Date: Sun, 29 Apr 2018 21:19:14 +0200 Subject: initial support for large objects This has gone through a lot of iterations on my branch, and I'm quite happy with the parts of the API that exist now. Test coverage can still be better, and will get better in the following commits. The API is not yet finished: I want to add Options arguments to Object.Upload(), Object.Copy(), Object.Move() and Object.Delete() that specify how each of these operations affect existing segments (and, later, also existing symlinks). For Upload(), uploading in segments shall become as easy as flipping a single switch. --- tests/largeobject_test.go | 362 ++++++++++++++++++++++++++++++++++++++++++++++ tests/object_test.go | 4 +- tests/shared_test.go | 13 ++ 3 files changed, 378 insertions(+), 1 deletion(-) create mode 100644 tests/largeobject_test.go (limited to 'tests') diff --git a/tests/largeobject_test.go b/tests/largeobject_test.go new file mode 100644 index 0000000..8e57e0d --- /dev/null +++ b/tests/largeobject_test.go @@ -0,0 +1,362 @@ +/****************************************************************************** +* +* Copyright 2018 Stefan Majewsky +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +******************************************************************************/ + +package tests + +import ( + "bytes" + "fmt" + "strings" + "testing" + + "github.com/majewsky/schwift" +) + +func foreachLargeObjectStrategy(action func(schwift.LargeObjectStrategy, string)) { + action(schwift.StaticLargeObject, "slo") + action(schwift.DynamicLargeObject, "dlo") +} + +func TestLargeObjectsBasic(t *testing.T) { + testWithContainer(t, func(c *schwift.Container) { + foreachLargeObjectStrategy(func(strategy schwift.LargeObjectStrategy, strategyStr string) { + + obj := c.Object(strategyStr + "-largeobject") + lo, err := obj.AsLargeObject() + expectSuccess(t, err) + + //Open fails when SegmentContainer is not set + _, err = lo.Open(schwift.OpenTruncate) + expectError(t, err, schwift.ErrNoContainerName.Error()) + + lo.SegmentContainer = c + lo.SegmentPrefix = strategyStr + "-segments/" + lo.Strategy = strategy + + segment1 := getRandomSegmentContent(128) + segment2 := getRandomSegmentContent(128) + segment3 := getRandomSegmentContent(128) + segment4 := getRandomSegmentContent(128) + + //basic write example + w, err := lo.Open(schwift.OpenTruncate) + expectSuccess(t, err) + if w == nil { + t.FailNow() + } + _, err = w.Write([]byte(segment1)) + expectSuccess(t, err) + _, err = w.Write([]byte(segment2)) + expectSuccess(t, err) + expectSuccess(t, w.Close()) + + expectObjectContent(t, obj, []byte(segment1+segment2)) + expectLargeObject(t, obj, []schwift.SegmentInfo{ + { + Object: c.Object(strategyStr + "-segments/0000000000000001"), + SizeBytes: 128, + Etag: etagOfString(segment1), + }, + { + Object: c.Object(strategyStr + "-segments/0000000000000002"), + SizeBytes: 128, + Etag: etagOfString(segment2), + }, + }) + + //basic append example + lo, err = obj.AsLargeObject() + expectSuccess(t, err) + expectLargeObjectSetup(t, lo, strategy, + fmt.Sprintf("%s/%s-segments/", c.Name(), strategyStr)) + w, err = lo.Open(schwift.OpenAppend) + expectSuccess(t, err) + if w == nil { + t.FailNow() + } + _, err = w.Write([]byte(segment3)) + expectSuccess(t, err) + _, err = w.Write([]byte(segment4)) + expectSuccess(t, err) + expectSuccess(t, w.Close()) + + expectObjectContent(t, obj, []byte(segment1+segment2+segment3+segment4)) + expectLargeObject(t, obj, []schwift.SegmentInfo{ + { + Object: c.Object(strategyStr + "-segments/0000000000000001"), + SizeBytes: 128, + Etag: etagOfString(segment1), + }, + { + Object: c.Object(strategyStr + "-segments/0000000000000002"), + SizeBytes: 128, + Etag: etagOfString(segment2), + }, + { + Object: c.Object(strategyStr + "-segments/0000000000000003"), + SizeBytes: 128, + Etag: etagOfString(segment3), + }, + { + Object: c.Object(strategyStr + "-segments/0000000000000004"), + SizeBytes: 128, + Etag: etagOfString(segment4), + }, + }) + + //basic truncate example + lo, err = obj.AsLargeObject() + expectSuccess(t, err) + expectLargeObjectSetup(t, lo, strategy, + fmt.Sprintf("%s/%s-segments/", c.Name(), strategyStr)) + w, err = lo.Open(schwift.OpenTruncate) + expectSuccess(t, err) + if w == nil { + t.FailNow() + } + + //verify that segments were deleted + iter := c.Objects() + iter.Prefix = lo.SegmentPrefix + names, err := iter.Collect() + expectSuccess(t, err) + expectObjectNames(t, names) + + _, err = w.Write([]byte(segment3)) + expectSuccess(t, err) + _, err = w.Write([]byte(segment4)) + expectSuccess(t, err) + expectSuccess(t, w.Close()) + + expectObjectContent(t, obj, []byte(segment3+segment4)) + expectLargeObject(t, obj, []schwift.SegmentInfo{ + { + Object: c.Object(strategyStr + "-segments/0000000000000001"), + SizeBytes: 128, + Etag: etagOfString(segment3), + }, + { + Object: c.Object(strategyStr + "-segments/0000000000000002"), + SizeBytes: 128, + Etag: etagOfString(segment4), + }, + }) + + }) + }) +} + +func TestOpenRegularObjectAsLargeObject(t *testing.T) { + testWithContainer(t, func(c *schwift.Container) { + o := c.Object("foo") + expectSuccess(t, o.Upload(bytes.NewReader(objectExampleContent), nil)) + _, err := o.AsLargeObject() + expectError(t, err, schwift.ErrNotLarge.Error()) + }) +} + +func TestSLOWithDataSegment(t *testing.T) { + testWithContainer(t, func(c *schwift.Container) { + o := c.Object("foo") + lo, err := o.AsLargeObject() + expectSuccess(t, err) + lo.SegmentContainer = c + lo.SegmentPrefix = "segments/" + lo.Strategy = schwift.StaticLargeObject + + segment1 := getRandomSegmentContent(128) + segment2 := getRandomSegmentContent(128) + w, err := lo.Open(schwift.OpenTruncate) + expectSuccess(t, err) + if w == nil { + t.FailNow() + } + _, err = w.Write([]byte(segment1)) + expectSuccess(t, err) + + dataSegment := schwift.SegmentInfo{Data: []byte("---")} + err = lo.AddSegment(dataSegment) + expectSuccess(t, err) + + _, err = w.Write([]byte(segment2)) + expectSuccess(t, err) + expectSuccess(t, w.Close()) + + expectObjectContent(t, o, []byte(segment1+string(dataSegment.Data)+segment2)) + expectLargeObject(t, o, []schwift.SegmentInfo{ + { + Object: c.Object("segments/0000000000000001"), + SizeBytes: 128, + Etag: etagOfString(segment1), + }, + dataSegment, + { + Object: c.Object("segments/0000000000000002"), + SizeBytes: 128, + Etag: etagOfString(segment2), + }, + }) + }) +} + +func TestSLOWithRangeSegments(t *testing.T) { + testWithContainer(t, func(c *schwift.Container) { + segmentStr := "XX" + segmentObj := c.Object("segment") + expectSuccess(t, segmentObj.Upload(bytes.NewReader([]byte(segmentStr)), nil)) + + o := c.Object("largeobject") + lo, err := o.AsLargeObject() + expectSuccess(t, err) + lo.SegmentContainer = c + lo.SegmentPrefix = "segments/" + lo.Strategy = schwift.StaticLargeObject + + //the large object is composed out of three ranges such that the "X" are precisely cut out of segmentStr + expectSuccess(t, lo.AddSegment(schwift.SegmentInfo{ + Object: segmentObj, + RangeLength: 5, + })) + expectSuccess(t, lo.AddSegment(schwift.SegmentInfo{ + Object: segmentObj, + RangeOffset: 6, + RangeLength: 5, + })) + expectSuccess(t, lo.AddSegment(schwift.SegmentInfo{ + Object: segmentObj, + RangeOffset: -1, + RangeLength: 5, + })) + expectSuccess(t, lo.WriteManifest(nil)) + + expectObjectContent(t, o, []byte( + strings.Replace(segmentStr, "X", "", -1), + )) + expectLargeObject(t, o, []schwift.SegmentInfo{ + { + Object: segmentObj, + Etag: etagOfString(segmentStr), + SizeBytes: uint64(len(segmentStr)), + RangeLength: 5, + }, + { + Object: segmentObj, + Etag: etagOfString(segmentStr), + SizeBytes: uint64(len(segmentStr)), + RangeOffset: 6, + RangeLength: 5, + }, + { + Object: segmentObj, + Etag: etagOfString(segmentStr), + SizeBytes: uint64(len(segmentStr)), + RangeOffset: -1, + RangeLength: 5, + }, + }) + }) +} + +func TestSLOGuessSegmentPrefix(t *testing.T) { + testWithContainer(t, func(c *schwift.Container) { + obj := c.Object("largeobject") + + //setup phase: create an SLO + lo, err := obj.AsLargeObject() + expectSuccess(t, err) + lo.SegmentContainer = c + lo.SegmentPrefix = "foo/bar/baz/" + w, err := lo.Open(schwift.OpenTruncate) + expectSuccess(t, err) + + segment1 := getRandomSegmentContent(128) + segment2 := getRandomSegmentContent(128) + _, err = w.Write([]byte(segment1)) + expectSuccess(t, err) + _, err = w.Write([]byte(segment2)) + expectSuccess(t, err) + expectSuccess(t, w.Close()) + + //now create a fresh SLO and check if it infers the correct SegmentPrefix + lo, err = obj.AsLargeObject() + expectSuccess(t, err) + expectString(t, lo.SegmentContainer.Name(), c.Name()) + expectString(t, lo.SegmentPrefix, "foo/bar/baz/") + }) +} + +func expectLargeObject(t *testing.T, obj *schwift.Object, expected []schwift.SegmentInfo) { + t.Helper() + expectObjectExistence(t, obj, true) + lo, err := obj.AsLargeObject() + expectSuccess(t, err) + if lo == nil { + t.FailNow() + } + + actual, err := lo.Segments() + expectSuccess(t, err) + if len(actual) != len(expected) { + t.Errorf("expected %s to have %d segments, got %d segments", + obj.FullName(), len(expected), len(actual)) + return + } + + for idx, as := range actual { + es := expected[idx] + if len(es.Data) > 0 { + //expecting data segment + if string(es.Data) != string(as.Data) { + t.Errorf("expected segments[%d].Data == %q, got %q", + idx, string(es.Data), string(as.Data)) + } + } else { + //expecting segment backed by object + if as.Object.FullName() != es.Object.FullName() { + t.Errorf("expected segments[%d].Object.FullName() == %q, got %q", + idx, es.Object.FullName(), as.Object.FullName()) + } + if es.SizeBytes != 0 && as.SizeBytes != es.SizeBytes { + t.Errorf("expected segments[%d].SizeBytes == %d, got %d", + idx, es.SizeBytes, as.SizeBytes) + } + if es.Etag != "" && as.Etag != es.Etag { + t.Errorf("expected segments[%d].Etag == %q, got %q", + idx, es.Etag, as.Etag) + } + } + } +} + +func expectLargeObjectSetup(t *testing.T, lo *schwift.LargeObject, strategy schwift.LargeObjectStrategy, segmentFullPrefix string) { + if strategy != lo.Strategy { + t.Errorf("expected %s to use LargeObjectStrategy %d, got %d", + lo.Object.FullName(), strategy, lo.Strategy) + } + + if lo.SegmentContainer == nil { + t.Errorf("expected %s to use segment container+prefix %q, got no container", + lo.Object.FullName(), segmentFullPrefix) + } else { + fullPrefix := lo.SegmentContainer.Name() + "/" + lo.SegmentPrefix + if fullPrefix != segmentFullPrefix { + t.Errorf("expected %s to use segment container+prefix %q, got %q", + lo.Object.FullName(), segmentFullPrefix, fullPrefix) + } + } +} diff --git a/tests/object_test.go b/tests/object_test.go index 1bbf953..39fdba2 100644 --- a/tests/object_test.go +++ b/tests/object_test.go @@ -225,5 +225,7 @@ func expectObjectContent(t *testing.T, obj *schwift.Object, expected []byte) { obj.Invalidate() hdr, err := obj.Headers() expectSuccess(t, err) - expectString(t, hdr.Etag().Get(), etagOf(expected)) + if !hdr.IsLargeObject() { + expectString(t, hdr.Etag().Get(), etagOf(expected)) + } } diff --git a/tests/shared_test.go b/tests/shared_test.go index c35951f..7867d70 100644 --- a/tests/shared_test.go +++ b/tests/shared_test.go @@ -114,6 +114,10 @@ func etagOf(buf []byte) string { return hex.EncodeToString(hash[:]) } +func etagOfString(buf string) string { + return etagOf([]byte(buf)) +} + func getRandomName() string { var buf [16]byte _, err := rand.Read(buf[:]) @@ -123,6 +127,15 @@ func getRandomName() string { return hex.EncodeToString(buf[:]) } +func getRandomSegmentContent(length int) string { + buf := make([]byte, length/2) + _, err := rand.Read(buf) + if err != nil { + panic(err.Error()) + } + return hex.EncodeToString(buf) +} + //////////////////////////////////////////////////////////////////////////////// func expectBool(t *testing.T, actual, expected bool) { -- cgit v1.2.3