updated the document

README.md

@@ -137,7 +137,7 @@ The following tools and libraries are required to build SentencePiece:
 
 On Ubuntu, the build tools can be installed with apt-get:
 ```
-% sudo apt-get install cmake pkg-config libgoogle-perftools-dev
+% sudo apt-get install cmake build-essential pkg-config libgoogle-perftools-dev
 ```
 ### Build and Install SentencePiece
 ```
@@ -165,8 +165,6 @@ On OSX/macOS, replace the last command with the following:
 * `--character_coverage`: amount of characters covered by the model, good defaults are: `0.9995` for languages with rich character set like Japanse or Chinese and `1.0` for other languages with small character set.
 * `--model_type`: model type. Choose from `unigram` (default), `bpe`, `char`, or `word`. The input sentence must be pretokenized when using `word` type.
 
-Note that `spm_train` loads only the first `--input_sentence_size` sentences (default value is 10M).
-
 Use `--help` flag to display all parameters for training.
 
 ### Encode raw text into sentence pieces/ids

src/filesystem.cc

@@ -27,7 +27,7 @@ class PosixReadableFile : public ReadableFile {
                 ? &std::cin
                 : new std::ifstream(WPATH(filename.data()),
                                     is_binary ? std::ios::binary | std::ios::in
-                                              : std::ios::out)) {
+                                              : std::ios::in)) {
     if (!*is_)
       status_ = util::StatusBuilder(util::error::NOT_FOUND)
                 << "\"" << filename.data() << "\": " << util::StrError(errno);

src/spec_parser.h

@@ -31,6 +31,7 @@ inline std::string PrintProto(const TrainerSpec &message) {
   os << "  split_by_unicode_script: " << message.split_by_unicode_script() << "\n";
   os << "  split_by_number: " << message.split_by_number() << "\n";
   os << "  split_by_whitespace: " << message.split_by_whitespace() << "\n";
+  os << "  treat_whitespace_as_suffix: " << message.treat_whitespace_as_suffix() << "\n";
   for (const auto &v : message.control_symbols())
     os << "  control_symbols: " << v << "\n";
   for (const auto &v : message.user_defined_symbols())
@@ -233,6 +234,15 @@ util::Status SentencePieceTrainer::SetProtoField(const std::string& name, const
     return util::OkStatus();
   }
 
+  if (name == "treat_whitespace_as_suffix") {
+    const auto &val = value;
+    bool v;
+    if (!string_util::lexical_cast(val.empty() ? "true" : val, &v))
+      return util::StatusBuilder(util::error::INVALID_ARGUMENT) << "cannot parse \"" << val << "\" as bool.";
+    message->set_treat_whitespace_as_suffix(v);
+    return util::OkStatus();
+  }
+
   if (name == "control_symbols") {
     for (const auto &val : string_util::Split(value, ",")) {
       message->add_control_symbols(val);

third_party/protobuf-lite/LICENSE

@@ -0,0 +1,32 @@
+Copyright 2008 Google Inc.  All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+    * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Code generated by the Protocol Buffer compiler is owned by the owner
+of the input file used when generating it.  This code is not
+standalone and requires a support library to be linked with it.  This
+support library is itself covered by the above license.