From 5e9bbd4d23a498cb0a8548d823bd2ec5d0d55be6 Mon Sep 17 00:00:00 2001 From: Daniel Fireman Date: Mon, 26 Feb 2018 21:37:05 -0300 Subject: [PATCH] Final API implementation of RawRead returning an io.ReadCloser (file like) object. #13 --- README.md | 4 ++- datapackage/package_test.go | 4 ++- datapackage/resource.go | 61 +++++++++++++++++++----------------- datapackage/resource_test.go | 10 ++++-- 4 files changed, 46 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 108517d..a18cc17 100644 --- a/README.md +++ b/README.md @@ -238,7 +238,9 @@ Once the data package is loaded, we could use the [Resource.RawRead](https://god ```go so := pkg.GetResource("schemaorg") -soContents, _ := so.RawRead() +rc, _ := so.RawRead() +defer rc.Close() +contents, _ := ioutil.ReadAll(rc) // Use contents. For instance, one could validate the JSON-LD schema and unmarshal it into a data structure. data := pkg.GetResource("data") diff --git a/datapackage/package_test.go b/datapackage/package_test.go index 0ccd1de..a74db55 100644 --- a/datapackage/package_test.go +++ b/datapackage/package_test.go @@ -73,7 +73,9 @@ func ExampleLoad_readRaw() { ioutil.WriteFile(resPath, resContent, 0666) pkg, _ := Load(descriptorPath, validator.InMemoryLoader()) - contents, _ := pkg.GetResource("res1").RawRead() + rc, _ := pkg.GetResource("res1").RawRead() + defer rc.Close() + contents, _ := ioutil.ReadAll(rc) fmt.Println(string(contents)) // Output: {"@context": {"@vocab": "http://schema.org/"}} } diff --git a/datapackage/resource.go b/datapackage/resource.go index 9dc6d9e..7b55529 100644 --- a/datapackage/resource.go +++ b/datapackage/resource.go @@ -194,15 +194,7 @@ func (r *Resource) GetTable(opts ...csv.CreationOpts) (table.Table, error) { return nil, fmt.Errorf("only csv and string is supported for inlining data") } } - buf, err := loadContents(r.basePath, r.path, csvLoadFunc) - if err != nil { - return nil, err - } - t, err := csv.NewTable(func() (io.ReadCloser, error) { return ioutil.NopCloser(bytes.NewReader(buf)), nil }, fullOpts...) - if err != nil { - return nil, err - } - return t, nil + return csv.NewTable(func() (io.ReadCloser, error) { return loadContents(r.basePath, r.path, csvLoadFunc) }, fullOpts...) } func csvLoadFunc(p string) func() (io.ReadCloser, error) { @@ -233,12 +225,7 @@ func binaryLoadFunc(p string) func() (io.ReadCloser, error) { if err != nil { return nil, err } - defer resp.Body.Close() - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - return ioutil.NopCloser(bytes.NewReader(b)), nil + return resp.Body, nil } } return func() (io.ReadCloser, error) { @@ -246,10 +233,31 @@ func binaryLoadFunc(p string) func() (io.ReadCloser, error) { } } -type loadFunc func(string) func() (io.ReadCloser, error) +type multiReadCloser struct { + io.Reader + rcs []io.ReadCloser +} + +func (m *multiReadCloser) Close() error { + var err error + for _, rc := range m.rcs { + if e := rc.Close(); e != nil { + err = e + } + } + return err +} + +func newMultiReadCloser(rcs []io.ReadCloser) io.ReadCloser { + readers := make([]io.Reader, len(rcs)) + for i := range rcs { + readers[i] = io.Reader(rcs[i]) + } + return &multiReadCloser{io.MultiReader(readers...), rcs} +} -func loadContents(basePath string, path []string, f loadFunc) ([]byte, error) { - var buf bytes.Buffer +func loadContents(basePath string, path []string, f func(string) func() (io.ReadCloser, error)) (io.ReadCloser, error) { + var rcs []io.ReadCloser for _, p := range path { if basePath != "" { p = joinPaths(basePath, p) @@ -258,17 +266,12 @@ func loadContents(basePath string, path []string, f loadFunc) ([]byte, error) { if err != nil { return nil, err } - defer rc.Close() - b, err := ioutil.ReadAll(rc) - if err != nil { - return nil, err - } - buf.Write(b) + rcs = append(rcs, rc) if len(path) > 1 { - buf.WriteRune('\n') + rcs = append(rcs, ioutil.NopCloser(bytes.NewReader([]byte{'\n'}))) } } - return buf.Bytes(), nil + return newMultiReadCloser(rcs), nil } func joinPaths(basePath, path string) string { @@ -289,11 +292,11 @@ func (r *Resource) ReadAll(opts ...csv.CreationOpts) ([][]string, error) { return t.ReadAll() } -// RawRead reads all resource contents and return it as byte slice. +// RawRead returns an io.ReaderCloser associated to the resource contents. // It can be used to access the content of non-tabular resources. -func (r *Resource) RawRead() ([]byte, error) { +func (r *Resource) RawRead() (io.ReadCloser, error) { if r.data != nil { - return []byte(r.data.(string)), nil + return ioutil.NopCloser(bytes.NewReader([]byte(r.data.(string)))), nil } return loadContents(r.basePath, r.path, binaryLoadFunc) } diff --git a/datapackage/resource_test.go b/datapackage/resource_test.go index 59fd5a6..89cff91 100644 --- a/datapackage/resource_test.go +++ b/datapackage/resource_test.go @@ -434,7 +434,10 @@ func TestResource_RawRead(t *testing.T) { }`, ts.URL) res, err := NewResourceFromString(resStr, validator.MustInMemoryRegistry()) is.NoErr(err) - contents, err := res.RawRead() + rc, err := res.RawRead() + is.NoErr(err) + defer rc.Close() + contents, err := ioutil.ReadAll(rc) is.NoErr(err) is.Equal(string(contents), "1234") }) @@ -446,7 +449,10 @@ func TestResource_RawRead(t *testing.T) { }` res, err := NewResourceFromString(resStr, validator.MustInMemoryRegistry()) is.NoErr(err) - contents, err := res.RawRead() + rc, err := res.RawRead() + is.NoErr(err) + defer rc.Close() + contents, err := ioutil.ReadAll(rc) is.NoErr(err) is.Equal(string(contents), "{\"foo\":\"1234\"}") })