Merge branch 'br-4.0' into generate-pb-option-parsers

mransan · Jun 10, 2024 · cea4460 · cea4460
2 parents c59e8e7 + 2e36f5d
commit cea4460
Show file tree

Hide file tree

Showing 46 changed files with 1,891 additions and 96 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,2 @@
+_opam
+_build
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -21,8 +21,8 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v2
-      #with:
-      #  submodules: true
+      with:
+        submodules: true
     - uses: ocaml/setup-ocaml@v2
       with:
         ocaml-compiler: ${{ matrix.ocaml-compiler }}
@@ -34,7 +34,29 @@ jobs:
     - run: opam install -t . --deps-only
     - run: opam exec -- dune build @install
     - run: opam exec -- dune runtest
-      #- run: opam exec -- make integration
+
+  integration:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - run: docker build -f dep/Dockerfile . --target=base
+    - run: docker build -f dep/Dockerfile . --target=run-the-damn-build
+
+  format:
+    name: format
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      #with:
+      #  submodules: true
+    - uses: ocaml/setup-ocaml@v2
+      with:
+        ocaml-compiler: 5.1
+        allow-prerelease-opam: true
+        dune-cache: true
+          #- run: sudo apt install protobuf-compiler libprotobuf-dev
     - run: opam install ocamlformat.0.24.1
     - run: opam exec -- dune build @fmt --auto-promote
     - run: git diff -q
diff --git a/Makefile.test b/Makefile.test
@@ -32,39 +32,42 @@ tests:
 
 # location of where the Google protoc compiler is installed
 PB_INSTALL ?= /usr/
-PB_HINC    = $(PB_INSTALL)/include
-PB_LINC    = $(PB_INSTALL)/lib
+PB_HINC    ?= $(PB_INSTALL)/include
+PB_LINC    ?= $(PB_INSTALL)/lib
 PROTOC     = $(PB_INSTALL)/bin/protoc
 
+PB_HINC_I = $(addprefix -I, $(PB_HINC))
+PB_LINC_L = $(addprefix -L, $(PB_LINC))
+
 export LD_LIBRARY_PATH=$(PB_LINC)
 
 $(OCAMLOPTIONS_HINC)/ocamloptions.pb.cc: $(OCAMLOPTIONS_HINC)/ocamloptions.proto
 	cd $(OCAMLOPTIONS_HINC) && \
-		$(PROTOC) --cpp_out ./ -I ./ -I $(abspath $(PB_HINC)) ocamloptions.proto
+		$(PROTOC) --cpp_out ./ -I ./ $(PB_HINC_I) ocamloptions.proto
 
 %_cpp.tsk: %_cpp.cpp %.pb.cc $(OCAMLOPTIONS_HINC)/ocamloptions.pb.cc
 	$(CXX) $(CPPFLAGS) $(LDFLAGS) \
-		-I ./ -I $(INTEGRATION_TESTS_DIR) -I $(OCAMLOPTIONS_HINC) -I $(PB_HINC) \
-		-L $(PB_LINC) -l protobuf \
+		-I ./ -I $(INTEGRATION_TESTS_DIR) -I $(OCAMLOPTIONS_HINC) $(PB_HINC_L) \
+		$(PB_LINC_L) $(PB_HINC_I) \
 		$? \
-		-o $@
+		-o $@ -l protobuf
 
 $(INTEGRATION_TESTS_DIR)/test10_cpp.tsk: \
 	$(INTEGRATION_TESTS_DIR)/test10_cpp.cpp \
 	$(INTEGRATION_TESTS_DIR)/test10.pb.cc \
 	$(INTEGRATION_TESTS_DIR)/test09.pb.cc
 	$(CXX) $(CPPFLAGS) $(LDFLAGS) \
-		-I ./ -I $(INTEGRATION_TESTS_DIR)  -I $(PB_HINC) \
-		-L $(PB_LINC) -l protobuf \
+		-I ./ -I $(INTEGRATION_TESTS_DIR)  \
+		$(PB_LINC_L) $(PB_HINC_I) \
 		$? \
-		-o $@
+		-o $@ -l protobuf
 
 .SECONDARY:
 
 %.pb.cc: %.proto
 	$(PROTOC) \
 		--cpp_out $(INTEGRATION_TESTS_DIR) \
-		-I $(PB_HINC) -I $(OCAMLOPTIONS_HINC) -I $(INTEGRATION_TESTS_DIR) \
+		$(PB_HINC_I) -I $(OCAMLOPTIONS_HINC) -I $(INTEGRATION_TESTS_DIR) \
 		$<
 
 

diff --git a/README.md b/README.md
@@ -24,7 +24,6 @@
 compiler**. See [here](https://github.com/mransan/bs-protobuf-demo) for complete example.
 
 
-
 ### A simple example
 
 > This example generates the binary encoding, if you are more interested in a **JavaScript** 

diff --git a/dep/Dockerfile b/dep/Dockerfile
@@ -0,0 +1,22 @@
+FROM ocaml/opam:debian-ocaml-4.14 AS base
+
+WORKDIR /build/
+RUN sudo chown -R opam: /build/
+RUN sudo apt install protobuf-compiler libprotobuf-dev -y
+COPY --chown=opam ./src/ ./src/
+COPY --chown=opam pbrt.opam pbrt_services.opam pbrt_yojson.opam ocaml-protoc.opam \
+  Makefile Makefile.test dune-project dune .ocamlformat \
+  .ocamlformat-ignore ./
+
+RUN opam pin -n .
+RUN opam depext -yt ocaml-protoc
+RUN opam install -t . --deps-only
+
+FROM base AS run-the-damn-build
+RUN opam exec -- dune build @install
+RUN opam exec -- dune runtest
+
+# integration tests
+RUN opam install ppx_deriving -y
+RUN opam exec -- make clean build integration PB_LINC=/usr/lib/x86_64-linux-gnu/ PB_HINC=/usr/include/
+
diff --git a/src/compilerlib/pb_codegen_all.ml b/src/compilerlib/pb_codegen_all.ml
@@ -48,7 +48,7 @@ let new_ocaml_mod ~proto_file_options ~proto_file_name () : ocaml_mod =
   in
 
   (* write preludes *)
-  F.line self.ml "[@@@ocaml.warning \"-27-30-39\"]";
+  F.line self.ml "[@@@ocaml.warning \"-27-30-39-44\"]";
   F.empty_line self.ml;
   print_ppx self.ml;
   F.empty_line self.mli;

diff --git a/src/compilerlib/pb_parsing_util.ml b/src/compilerlib/pb_parsing_util.ml
@@ -328,7 +328,7 @@ let finalize_syntax3 proto =
                 | `Bool ->
                   let field_options =
                     Pb_raw_option.(
-                      add field_options [ Simple_name "packed" ]
+                      add_or_replace field_options [ Simple_name "packed" ]
                         (Scalar_value (Constant_bool true)))
                   in
                   { field with Pt.field_options }

diff --git a/src/compilerlib/pb_raw_option.ml b/src/compilerlib/pb_raw_option.ml
@@ -48,6 +48,15 @@ let option_name_equal = Pb_util.List.equal option_name_part_equal
 let empty = []
 let add option_set option_name value = (option_name, value) :: option_set
 
+let add_or_replace option_set option_name value =
+  let option_set =
+    List.filter
+      (fun (option_name', _) ->
+        not (option_name_equal option_name' option_name))
+      option_set
+  in
+  add option_set option_name value
+
 let get t option_name =
   match List.find (fun (other, _) -> option_name_equal option_name other) t with
   | _, c -> Some c

diff --git a/src/compilerlib/pb_raw_option.mli b/src/compilerlib/pb_raw_option.mli
@@ -63,6 +63,7 @@ type set = t list
 val stringify_option_name : option_name -> string
 val empty : set
 val add : set -> option_name -> Pb_option.value -> set
+val add_or_replace : set -> option_name -> Pb_option.value -> set
 
 val merge : set -> set -> set
 (** [merge s1 s2] adds all the options from [s2] to [s1]. This means

diff --git a/src/dune b/src/dune
@@ -1,3 +1,5 @@
 (env
  (_
   (flags :standard -warn-error -a+8 -w +a-4-40-41-42-44-48-70)))
+
+(data_only_dirs runtime-bs)
diff --git a/src/examples/build_server.ml.expected b/src/examples/build_server.ml.expected
@@ -1,4 +1,4 @@
-[@@@ocaml.warning "-27-30-39"]
+[@@@ocaml.warning "-27-30-39-44"]
 
 type file_path = {
   path : string;

diff --git a/src/examples/calculator.ml.expected b/src/examples/calculator.ml.expected
@@ -1,4 +1,4 @@
-[@@@ocaml.warning "-27-30-39"]
+[@@@ocaml.warning "-27-30-39-44"]
 
 type div_by_zero = unit
 

diff --git a/src/examples/example01.ml.expected b/src/examples/example01.ml.expected
@@ -1,4 +1,4 @@
-[@@@ocaml.warning "-27-30-39"]
+[@@@ocaml.warning "-27-30-39-44"]
 
 type person = {
   name : string;

diff --git a/src/examples/example03.ml.expected b/src/examples/example03.ml.expected
@@ -1,4 +1,4 @@
-[@@@ocaml.warning "-27-30-39"]
+[@@@ocaml.warning "-27-30-39-44"]
 
 type string_some_none = unit
 

diff --git a/src/examples/example04.ml.expected b/src/examples/example04.ml.expected
@@ -1,4 +1,4 @@
-[@@@ocaml.warning "-27-30-39"]
+[@@@ocaml.warning "-27-30-39-44"]
 
 type int_list_nil = unit
 

diff --git a/src/examples/example05.ml.expected b/src/examples/example05.ml.expected
@@ -1,4 +1,4 @@
-[@@@ocaml.warning "-27-30-39"]
+[@@@ocaml.warning "-27-30-39-44"]
 
 type person = {
   name : string;

diff --git a/src/examples/file_server.ml.expected b/src/examples/file_server.ml.expected
@@ -1,4 +1,4 @@
-[@@@ocaml.warning "-27-30-39"]
+[@@@ocaml.warning "-27-30-39-44"]
 
 type file_chunk = {
   path : string;

diff --git a/src/examples/oneof.ml.expected b/src/examples/oneof.ml.expected
@@ -1,4 +1,4 @@
-[@@@ocaml.warning "-27-30-39"]
+[@@@ocaml.warning "-27-30-39-44"]
 
 type patch_copy = {
   start : int64;

diff --git a/src/examples/orgchart.ml.expected b/src/examples/orgchart.ml.expected
@@ -1,4 +1,4 @@
-[@@@ocaml.warning "-27-30-39"]
+[@@@ocaml.warning "-27-30-39-44"]
 
 type person = {
   name : string;

diff --git a/src/runtime/stubs.c b/src/runtime/stubs.c
@@ -65,7 +65,7 @@ CAMLprim value caml_pbrt_varint_byte(value _str, value _idx, value _i) {
   CAMLparam3(_str, _idx, _i);
   char *str = Bytes_val(_str);
   int idx = Int_val(_idx);
-  int64_t i = Int64_val(_idx);
+  int64_t i = Int64_val(_i);
   pbrt_varint(str + idx, i);
   CAMLreturn(Val_unit);
 }
diff --git a/src/tests/benchmark/benchmark_single_ml.ml b/src/tests/benchmark/benchmark_single_ml.ml
@@ -45,8 +45,12 @@ let make_test_requests ~number_of_sample test_ids =
       let file_name =
         Printf.sprintf "test_%s_%i.data" (string_of_test_id test_id) difficulty
       in
-      let encode = Bench_t.{ type_ = Encode difficulty; file_name; test_id } in
-      let decode = Bench_t.{ type_ = Decode; file_name; test_id } in
+      let encode =
+        Bench_t.{ type_ = { t = Some (Encode difficulty) }; file_name; test_id }
+      in
+      let decode =
+        Bench_t.{ type_ = { t = Some Decode }; file_name; test_id }
+      in
 
       let init_cases, test_cases = acc in
       let test_cases = Util.append_n test_cases encode number_of_sample in
@@ -150,10 +154,10 @@ let print_compare_type ~metric responses test_ids =
             (fun data ->
               let open Bench_t in
               match data, metric with
-              | Encode { encode_time; _ }, `Encode_time ->
+              | Some (Encode { encode_time; _ }), `Encode_time ->
                 sum := !sum +. encode_time;
                 incr counter
-              | Decode { decode_time; _ }, `Decode_time ->
+              | Some (Decode { decode_time; _ }), `Decode_time ->
                 sum := !sum +. decode_time;
                 incr counter
               | _ -> ())

diff --git a/src/tests/benchmark/ocaml_test_runner.ml b/src/tests/benchmark/ocaml_test_runner.ml
@@ -27,8 +27,9 @@ end
 
 module Make (T : T_sig) : Runner_sig = struct
   let run { Benchmark.type_; file_name; test_id } =
-    match type_ with
-    | Benchmark.Encode difficulty ->
+    match type_.t with
+    | None -> failwith "missing benchmark"
+    | Some (Encode difficulty) ->
       let t0 = Unix.gettimeofday () in
 
       let encoder = Pbrt.Encoder.create () in
@@ -53,8 +54,8 @@ module Make (T : T_sig) : Runner_sig = struct
           }
       in
 
-      Benchmark.{ difficulty_size; test_id; data = Encode encode_data }
-    | Benchmark.Decode ->
+      Benchmark.{ difficulty_size; test_id; data = Some (Encode encode_data) }
+    | Some Decode ->
       let t0 = Unix.gettimeofday () in
       let ic = open_in file_name in
       let len = in_channel_length ic in
@@ -69,11 +70,12 @@ module Make (T : T_sig) : Runner_sig = struct
           difficulty_size = T.difficulty_size v;
           test_id;
           data =
-            Decode
-              {
-                Benchmark.from_file_time = t1 -. t0;
-                Benchmark.decode_time = t2 -. t1;
-              };
+            Some
+              (Decode
+                 {
+                   Benchmark.from_file_time = t1 -. t0;
+                   Benchmark.decode_time = t2 -. t1;
+                 });
         }
 end
 (* Make *)
diff --git a/src/tests/expectation/dune b/src/tests/expectation/dune
@@ -1,5 +1,13 @@
 (test
- (name it_compiles)
+ (name test_option_parsing)
+ (modules Test_option_parsing)
+ (libraries pbrt ocaml-protoc.compiler-lib)
+ (package ocaml-protoc)
+ (flags :standard -open Ocaml_protoc_compiler_lib))
+
+(test
+ (name test_option_compilation)
+ (modules Test_option_compilation)
  (libraries pbrt ocaml-protoc.compiler-lib)
  (package ocaml-protoc)
  (flags :standard -open Ocaml_protoc_compiler_lib)

diff --git a/src/tests/expectation/option_processing.ml.expected b/src/tests/expectation/option_processing.ml.expected
@@ -1,4 +1,4 @@
-[@@@ocaml.warning "-27-30-39"]
+[@@@ocaml.warning "-27-30-39-44"]
 
 type payment_system =
   | Cash