Add: initial UTF-8 support on Windows with stringi #5

DESCRIPTION

@@ -9,13 +9,13 @@ Description: RE2 is a primarily DFA based regexp engine from Google that is very
 License: BSD_3_clause + file LICENSE
 LazyData: TRUE
 Imports:
-    Rcpp (>= 0.12.2)
+    Rcpp (>= 0.12.2), stringi
 LinkingTo: Rcpp
 Suggests:
     knitr (>= 1.12.3),testthat,
     rmarkdown (>= 0.9.5),microbenchmark,rex
 Enhances:
-    directlabels, ggplot2 ,stringi
+    directlabels, ggplot2
 URL: https://github.com/qinwf/re2r/
 BugReports: https://github.com/qinwf/re2r/issues
 VignetteBuilder: knitr

NAMESPACE

@@ -15,4 +15,5 @@ export(re2_extract)
 export(re2_match)
 export(re2_replace)
 import(Rcpp)
+importFrom(stringi,stri_enc_toutf8)
 useDynLib(re2r)

R/compile.R

@@ -108,7 +108,7 @@ re2 = function(pattern,
     #     pattern = enc2utf8(pattern)
     # }
     regexp = cpp_re2_compile(
-        pattern,
+        stri_enc_toutf8(pattern),
         log_errors_value = FALSE,
         utf_8_value = utf_8,
         case_sensitive_value = case_sensitive,
@@ -178,10 +178,7 @@ get_named_groups = function(regexp) {
 #' @return quoted string
 #' @export
 quote_meta = function(unquoted) {
-    # if (check_windows_strings(unquoted)) {
-    #     unquoted = enc2utf8(unquoted)
-    # }
-    res = cpp_quote_meta(unquoted)
+    res = cpp_quote_meta(stri_enc_toutf8(unquoted))
     # if (update_windows_strings()) {
     #     Encoding(res) = "UTF-8"
     # }

R/extract.R

@@ -48,5 +48,5 @@ re2_extract = function(input, pattern, rewrite = "\\1", ...) {
     if (is.character(pattern)) {
         pattern = re2(pattern, ...)
     }
-    cpp_extract(input, pattern, rewrite)
+    cpp_extract(stri_enc_toutf8(input), pattern, stri_enc_toutf8(rewrite))
 }

R/match.R

@@ -69,5 +69,5 @@ re2_match = function(input,
     if (is.character(pattern)) {
         pattern = re2(pattern, ...)
     }
-    cpp_match(input, pattern, value, anchor, all)
+    cpp_match(stri_enc_toutf8(input), pattern, value, anchor, all)
 }

R/replace.R

@@ -52,5 +52,5 @@ re2_replace = function(input, pattern, rewrite,  all = FALSE, ...) {
     if (is.character(pattern)) {
         pattern = re2(pattern, ...)
     }
-    cpp_replace(input, pattern, rewrite,  all)
+    cpp_replace(stri_enc_toutf8(input), pattern, stri_enc_toutf8(rewrite),  all)
 }

R/zzz.R

@@ -32,4 +32,5 @@
 
 ##' @useDynLib re2r
 ##' @import Rcpp
+##' @importFrom stringi stri_enc_toutf8
 NULL

tests/testthat/test-unicode.R

@@ -0,0 +1,6 @@
+context("Unicode")
+
+test_that("unicode match",{
+    x <- stringi::stri_conv("a\u6587bc", "UTF-8", "")
+    expect_true(re2_match(x,"\u6587"))
+})