Add UTF8Encoding func

wabarc · Jul 6, 2021 · 6ba9e89 · 6ba9e89
1 parent e0fe0a8
commit 6ba9e89
Show file tree

Hide file tree

Showing 3 changed files with 44 additions and 1 deletion.
diff --git a/go.mod b/go.mod
@@ -2,4 +2,8 @@ module github.com/wabarc/helper
 
 go 1.15
 
-require mvdan.cc/xurls/v2 v2.2.0
+require (
+	golang.org/x/net v0.0.0-20210614182718-04defd469f4e
+	golang.org/x/text v0.3.6
+	mvdan.cc/xurls/v2 v2.2.0
+)
diff --git a/go.sum b/go.sum
@@ -2,6 +2,14 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
+golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=

diff --git a/string.go b/string.go
@@ -5,8 +5,15 @@
 package helper // import "github.com/wabarc/helper"
 
 import (
+	"bufio"
+	"io"
 	"math/rand"
+	"strings"
 	"time"
+
+	"golang.org/x/net/html/charset"
+	"golang.org/x/text/encoding"
+	"golang.org/x/text/transform"
 )
 
 func init() {
@@ -31,3 +38,27 @@ func RandString(length int, letter string) string {
 
 	return string(bytes)
 }
+
+func UTF8Encoding(s string) (r io.Reader, err error) {
+	buf := strings.NewReader(s)
+	e, name, err := determineEncodingFromReader(buf)
+	if err != nil {
+		return
+	}
+	rd, err := charset.NewReader(buf, name)
+	if err != nil {
+		return
+	}
+	r = transform.NewReader(rd, e.NewDecoder())
+	return
+}
+
+func determineEncodingFromReader(r io.Reader) (e encoding.Encoding, name string, err error) {
+	buf, err := bufio.NewReader(r).Peek(1024)
+	if err != nil {
+		return
+	}
+
+	e, name, _ = charset.DetermineEncoding(buf, "")
+	return
+}