Skip to content

Commit

Permalink
Schema default not taken into account when writing record (#18)
Browse files Browse the repository at this point in the history 
* Schema default not taken into account when writing record
* Add test case for default value
* Fix other tests after [FX] primitive types should serialize with type attribute
  • Loading branch information
@axelrtgs
axelrtgs authored Aug 10, 2022
1 parent 0eadbc1 commit 5936698
Show file tree
Hide file tree
Showing 3 changed files with 34 additions and 31 deletions.
6 changes: 4 additions & 2 deletions lib/avro/datum.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,10 @@ private function write_fixed($writers_schema, $datum, $encoder)
private function write_record($writers_schema, $datum, $encoder)
{
foreach ($writers_schema->fields() as $field)
$this->write_data($field->type(), $datum[$field->name()], $encoder);
{
$value = isset($datum[$field->name()]) ? $datum[$field->name()] : $field->default_value();
$this->write_data($field->type(), $value, $encoder);
}
}

/**#@-*/
Expand Down Expand Up @@ -1105,4 +1108,3 @@ private function seek($offset, $whence)
return $this->io->seek($offset, $whence);
}
}

5 changes: 3 additions & 2 deletions test/DataFileTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,11 +229,12 @@ public function test_differing_schemas_with_primitives($codec)
"fields" : [
{"name": "username", "type": "string"},
{"name": "age", "type": "int"},
{"name": "verified", "type": "boolean", "default": "false"}
{"name": "verified", "type": "boolean", "default": false}
]}
JSON;
$data = array(array('username' => 'john', 'age' => 25, 'verified' => true),
array('username' => 'ryan', 'age' => 23, 'verified' => false));
array('username' => 'ryan', 'age' => 23, 'verified' => false),
array('username' => 'bill', 'age' => 35));
$dw = AvroDataIO::open_file($data_file, 'w', $writer_schema, $codec);
foreach ($data as $datum)
{
Expand Down
54 changes: 27 additions & 27 deletions test/SchemaTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ protected static function make_primitive_examples()
'bytes', 'string')
as $type)
{
$examples []= new SchemaExample(sprintf('"%s"', $type), true);
$examples []= new SchemaExample(sprintf('"%s"', $type), true, sprintf('{"type":"%s"}', $type));
$examples []= new SchemaExample(sprintf('{"type": "%s"}', $type), true, sprintf('{"type":"%s"}', $type));
}
return $examples;
Expand All @@ -84,21 +84,21 @@ protected static function make_examples()
self::make_primitive_examples());

$array_examples = array(
new SchemaExample('{"type": "array", "items": "long"}', true),
new SchemaExample('{"type": "array", "items": "long"}', true, '{"type":"array","items":{"type":"long"}}'),
new SchemaExample('
{"type": "array",
"items": {"type": "enum", "name": "Test", "symbols": ["A", "B"]}}
', true));
', true, '{"type":"array","items":{"type":"enum","name":"Test","symbols":["A","B"]}}'));

$map_examples = array(
new SchemaExample('{"type": "map", "values": "long"}', true),
new SchemaExample('{"type": "map", "values": "long"}', true, '{"type":"map","values":{"type":"long"}}'),
new SchemaExample('
{"type": "map",
"values": {"type": "enum", "name": "Test", "symbols": ["A", "B"]}}
', true));
', true, ));

$union_examples = array(
new SchemaExample('["string", "null", "long"]', true),
new SchemaExample('["string", "null", "long"]', true, '[{"type":"string"},{"type":"null"},{"type":"long"}]'),
new SchemaExample('["null", "null"]', false),
new SchemaExample('["long", "long"]', false),
new SchemaExample('
Expand All @@ -111,7 +111,7 @@ protected static function make_examples()
new SchemaExample('["long",
{"type": "array", "items": "long"},
{"type": "map", "values": "long"},
"int"]', true),
"int"]', true, '[{"type":"long"},{"type":"array","items":{"type":"long"}},{"type":"map","values":{"type":"long"}},{"type":"int"}]'),
new SchemaExample('["long",
["string", "null"],
"int"]', false),
Expand All @@ -127,7 +127,7 @@ protected static function make_examples()
{"name": "foo", "type":"fixed",
"size":16},
{"name": "baz", "type":"enum", "symbols":["A", "B", "C"]}
]', true, '["null","boolean","int","long","float","double","string","bytes",{"type":"array","items":"int"},{"type":"map","values":"int"},{"type":"record","name":"bar","fields":[{"name":"label","type":"string"}]},{"type":"fixed","name":"foo","size":16},{"type":"enum","name":"baz","symbols":["A","B","C"]}]'),
]', true, '[{"type":"null"},{"type":"boolean"},{"type":"int"},{"type":"long"},{"type":"float"},{"type":"double"},{"type":"string"},{"type":"bytes"},{"type":"array","items":{"type":"int"}},{"type":"map","values":{"type":"int"}},{"type":"record","name":"bar","fields":[{"name":"label","type":{"type":"string"}}]},{"type":"fixed","name":"foo","size":16},{"type":"enum","name":"baz","symbols":["A","B","C"]}]'),
new SchemaExample('
[{"name":"subtract", "namespace":"com.example",
"type":"record",
Expand All @@ -138,7 +138,7 @@ protected static function make_examples()
"fields":[{"name":"quotient", "type":"int"},
{"name":"dividend", "type":"int"}]},
{"type": "array", "items": "string"}]
', true, '[{"type":"record","name":"subtract","namespace":"com.example","fields":[{"name":"minuend","type":"int"},{"name":"subtrahend","type":"int"}]},{"type":"record","name":"divide","namespace":"com.example","fields":[{"name":"quotient","type":"int"},{"name":"dividend","type":"int"}]},{"type":"array","items":"string"}]'),
', true, '[{"type":"record","name":"subtract","namespace":"com.example","fields":[{"name":"minuend","type":{"type":"int"}},{"name":"subtrahend","type":{"type":"int"}}]},{"type":"record","name":"divide","namespace":"com.example","fields":[{"name":"quotient","type":{"type":"int"}},{"name":"dividend","type":{"type":"int"}}]},{"type":"array","items":{"type":"string"}}]'),
);

$fixed_examples = array(
Expand Down Expand Up @@ -223,32 +223,32 @@ protected static function make_examples()
"name": "Test",
"fields": [{"name": "f",
"type": "long"}]}
', true);
', true, '{"type":"record","name":"Test","fields":[{"name":"f","type":{"type":"long"}}]}');
$record_examples []= new SchemaExample('
{"type": "error",
"name": "Test",
"fields": [{"name": "f",
"type": "long"}]}
', true);
', true, '{"type":"error","name":"Test","fields":[{"name":"f","type":{"type":"long"}}]}');
$record_examples []= new SchemaExample('
{"type": "record",
"name": "Node",
"fields": [{"name": "label", "type": "string"},
{"name": "children",
"type": {"type": "array", "items": "Node"}}]}
', true);
', true, '{"type":"record","name":"Node","fields":[{"name":"label","type":{"type":"string"}},{"name":"children","type":{"type":"array","items":"Node"}}]}');
$record_examples []= new SchemaExample('
{"type": "record",
"name": "ListLink",
"fields": [{"name": "car", "type": "int"},
{"name": "cdr", "type": "ListLink"}]}
', true);
', true, '{"type":"record","name":"ListLink","fields":[{"name":"car","type":{"type":"int"}},{"name":"cdr","type":"ListLink"}]}');
$record_examples []= new SchemaExample('
{"type": "record",
"name": "Lisp",
"fields": [{"name": "value",
"type": ["null", "string"]}]}
', true);
', true, '{"type":"record","name":"Lisp","fields":[{"name":"value","type":[{"type":"null"},{"type":"string"}]}]}');
$record_examples []= new SchemaExample('
{"type": "record",
"name": "Lisp",
Expand All @@ -258,7 +258,7 @@ protected static function make_examples()
"name": "Cons",
"fields": [{"name": "car", "type": "string"},
{"name": "cdr", "type": "string"}]}]}]}
', true);
', true, '{"type":"record","name":"Lisp","fields":[{"name":"value","type":[{"type":"null"},{"type":"string"},{"type":"record","name":"Cons","fields":[{"name":"car","type":{"type":"string"}},{"name":"cdr","type":{"type":"string"}}]}]}]}');
$record_examples []= new SchemaExample('
{"type": "record",
"name": "Lisp",
Expand All @@ -268,7 +268,7 @@ protected static function make_examples()
"name": "Cons",
"fields": [{"name": "car", "type": "Lisp"},
{"name": "cdr", "type": "Lisp"}]}]}]}
', true);
', true, '{"type":"record","name":"Lisp","fields":[{"name":"value","type":[{"type":"null"},{"type":"string"},{"type":"record","name":"Cons","fields":[{"name":"car","type":"Lisp"},{"name":"cdr","type":"Lisp"}]}]}]}');
$record_examples []= new SchemaExample('
{"type": "record",
"name": "HandshakeRequest",
Expand All @@ -277,7 +277,7 @@ protected static function make_examples()
"type": {"type": "fixed", "name": "MD5", "size": 16}},
{"name": "meta",
"type": ["null", {"type": "map", "values": "bytes"}]}]}
', true);
', true, '{"type":"record","name":"HandshakeRequest","namespace":"org.apache.avro.ipc","fields":[{"name":"clientHash","type":{"type":"fixed","name":"MD5","size":16}},{"name":"meta","type":[{"type":"null"},{"type":"map","values":{"type":"bytes"}}]}]}');
$record_examples []= new SchemaExample('
{"type": "record",
"name": "HandshakeRequest",
Expand All @@ -288,7 +288,7 @@ protected static function make_examples()
{"name": "serverHash", "type": "MD5"},
{"name": "meta",
"type": ["null", {"type": "map", "values": "bytes"}]}]}
', true);
', true, '{"type":"record","name":"HandshakeRequest","namespace":"org.apache.avro.ipc","fields":[{"name":"clientHash","type":{"type":"fixed","name":"MD5","size":16}},{"name":"clientProtocol","type":[{"type":"null"},{"type":"string"}]},{"name":"serverHash","type":"MD5"},{"name":"meta","type":[{"type":"null"},{"type":"map","values":{"type":"bytes"}}]}]}');
$record_examples []= new SchemaExample('
{"type": "record",
"name": "HandshakeResponse",
Expand All @@ -304,7 +304,7 @@ protected static function make_examples()
{"name": "meta",
"type": ["null", {"type": "map", "values": "bytes"}]}]}
', true,
'{"type":"record","name":"HandshakeResponse","namespace":"org.apache.avro.ipc","fields":[{"name":"match","type":{"type":"enum","name":"HandshakeMatch","symbols":["BOTH","CLIENT","NONE"]}},{"name":"serverProtocol","type":["null","string"]},{"name":"serverHash","type":["null",{"type":"fixed","name":"MD5","size":16}]},{"name":"meta","type":["null",{"type":"map","values":"bytes"}]}]}'
'{"type":"record","name":"HandshakeResponse","namespace":"org.apache.avro.ipc","fields":[{"name":"match","type":{"type":"enum","name":"HandshakeMatch","symbols":["BOTH","CLIENT","NONE"]}},{"name":"serverProtocol","type":[{"type":"null"},{"type":"string"}]},{"name":"serverHash","type":[{"type":"null"},{"type":"fixed","name":"MD5","size":16}]},{"name":"meta","type":[{"type":"null"},{"type":"map","values":{"type":"bytes"}}]}]}'
);
$record_examples []= new SchemaExample('{"type": "record",
"namespace": "org.apache.avro",
Expand All @@ -326,7 +326,7 @@ protected static function make_examples()
"name": "children"}],
"type": "record",
"name": "Node"},
"name": "recordField"}]}', true, '{"type":"record","name":"Interop","namespace":"org.apache.avro","fields":[{"name":"enumField","type":{"type":"enum","name":"Kind","symbols":["A","B","C"]}},{"name":"recordField","type":{"type":"record","name":"Node","fields":[{"name":"label","type":"string"},{"name":"children","type":{"type":"array","items":"Node"}}]}}]}');
"name": "recordField"}]}', true, '{"type":"record","name":"Interop","namespace":"org.apache.avro","fields":[{"name":"enumField","type":{"type":"enum","name":"Kind","symbols":["A","B","C"]}},{"name":"recordField","type":{"type":"record","name":"Node","fields":[{"name":"label","type":{"type":"string"}},{"name":"children","type":{"type":"array","items":"Node"}}]}}]}');

$record_examples []= new SchemaExample('
{"type": "record",
Expand Down Expand Up @@ -366,9 +366,9 @@ protected static function make_examples()
"type": {"type": "array",
"items": "Node"}}]}}]}
', true,
'{"type":"record","name":"Interop","namespace":"org.apache.avro","fields":[{"name":"intField","type":"int"},{"name":"longField","type":"long"},{"name":"stringField","type":"string"},{"name":"boolField","type":"boolean"},{"name":"floatField","type":"float"},{"name":"doubleField","type":"double"},{"name":"bytesField","type":"bytes"},{"name":"nullField","type":"null"},{"name":"arrayField","type":{"type":"array","items":"double"}},{"name":"mapField","type":{"type":"map","values":{"type":"record","name":"Foo","fields":[{"name":"label","type":"string"}]}}},{"name":"unionField","type":["boolean","double",{"type":"array","items":"bytes"}]},{"name":"enumField","type":{"type":"enum","name":"Kind","symbols":["A","B","C"]}},{"name":"fixedField","type":{"type":"fixed","name":"MD5","size":16}},{"name":"recordField","type":{"type":"record","name":"Node","fields":[{"name":"label","type":"string"},{"name":"children","type":{"type":"array","items":"Node"}}]}}]}');
'{"type":"record","name":"Interop","namespace":"org.apache.avro","fields":[{"name":"intField","type":{"type":"int"}},{"name":"longField","type":{"type":"long"}},{"name":"stringField","type":{"type":"string"}},{"name":"boolField","type":{"type":"boolean"}},{"name":"floatField","type":{"type":"float"}},{"name":"doubleField","type":{"type":"double"}},{"name":"bytesField","type":{"type":"bytes"}},{"name":"nullField","type":{"type":"null"}},{"name":"arrayField","type":{"type":"array","items":{"type":"double"}}},{"name":"mapField","type":{"type":"map","values":{"type":"record","name":"Foo","fields":[{"name":"label","type":{"type":"string"}}]}}},{"name":"unionField","type":[{"type":"boolean"},{"type":"double"},{"type":"array","items":{"type":"bytes"}}]},{"name":"enumField","type":{"type":"enum","name":"Kind","symbols":["A","B","C"]}},{"name":"fixedField","type":{"type":"fixed","name":"MD5","size":16}},{"name":"recordField","type":{"type":"record","name":"Node","fields":[{"name":"label","type":{"type":"string"}},{"name":"children","type":{"type":"array","items":"Node"}}]}}]}');
$record_examples []= new SchemaExample('{"type": "record", "namespace": "org.apache.avro", "name": "Interop", "fields": [{"type": "int", "name": "intField"}, {"type": "long", "name": "longField"}, {"type": "string", "name": "stringField"}, {"type": "boolean", "name": "boolField"}, {"type": "float", "name": "floatField"}, {"type": "double", "name": "doubleField"}, {"type": "bytes", "name": "bytesField"}, {"type": "null", "name": "nullField"}, {"type": {"items": "double", "type": "array"}, "name": "arrayField"}, {"type": {"type": "map", "values": {"fields": [{"type": "string", "name": "label"}], "type": "record", "name": "Foo"}}, "name": "mapField"}, {"type": ["boolean", "double", {"items": "bytes", "type": "array"}], "name": "unionField"}, {"type": {"symbols": ["A", "B", "C"], "type": "enum", "name": "Kind"}, "name": "enumField"}, {"type": {"type": "fixed", "name": "MD5", "size": 16}, "name": "fixedField"}, {"type": {"fields": [{"type": "string", "name": "label"}, {"type": {"items": "org.apache.avro.Node", "type": "array"}, "name": "children"}], "type": "record", "name": "Node"}, "name": "recordField"}]}
', true, '{"type":"record","name":"Interop","namespace":"org.apache.avro","fields":[{"name":"intField","type":"int"},{"name":"longField","type":"long"},{"name":"stringField","type":"string"},{"name":"boolField","type":"boolean"},{"name":"floatField","type":"float"},{"name":"doubleField","type":"double"},{"name":"bytesField","type":"bytes"},{"name":"nullField","type":"null"},{"name":"arrayField","type":{"type":"array","items":"double"}},{"name":"mapField","type":{"type":"map","values":{"type":"record","name":"Foo","fields":[{"name":"label","type":"string"}]}}},{"name":"unionField","type":["boolean","double",{"type":"array","items":"bytes"}]},{"name":"enumField","type":{"type":"enum","name":"Kind","symbols":["A","B","C"]}},{"name":"fixedField","type":{"type":"fixed","name":"MD5","size":16}},{"name":"recordField","type":{"type":"record","name":"Node","fields":[{"name":"label","type":"string"},{"name":"children","type":{"type":"array","items":"Node"}}]}}]}');
', true, '{"type":"record","name":"Interop","namespace":"org.apache.avro","fields":[{"name":"intField","type":{"type":"int"}},{"name":"longField","type":{"type":"long"}},{"name":"stringField","type":{"type":"string"}},{"name":"boolField","type":{"type":"boolean"}},{"name":"floatField","type":{"type":"float"}},{"name":"doubleField","type":{"type":"double"}},{"name":"bytesField","type":{"type":"bytes"}},{"name":"nullField","type":{"type":"null"}},{"name":"arrayField","type":{"type":"array","items":{"type":"double"}}},{"name":"mapField","type":{"type":"map","values":{"type":"record","name":"Foo","fields":[{"name":"label","type":{"type":"string"}}]}}},{"name":"unionField","type":[{"type":"boolean"},{"type":"double"},{"type":"array","items":{"type":"bytes"}}]},{"name":"enumField","type":{"type":"enum","name":"Kind","symbols":["A","B","C"]}},{"name":"fixedField","type":{"type":"fixed","name":"MD5","size":16}},{"name":"recordField","type":{"type":"record","name":"Node","fields":[{"name":"label","type":{"type":"string"}},{"name":"children","type":{"type":"array","items":"Node"}}]}}]}');
$record_examples []= new SchemaExample('
{"type": "record",
"name": "ipAddr",
Expand Down Expand Up @@ -404,7 +404,7 @@ protected static function make_examples()
"fields":[{"name":"bar", "type":"int", "order":"ascending", "default":1}]}
',
true,
'{"type":"record","name":"foo","doc":"doc string","fields":[{"name":"bar","type":"int","default":1,"order":"ascending"}]}');
'{"type":"record","name":"foo","doc":"doc string","fields":[{"name":"bar","type":{"type":"int"},"default":1,"order":"ascending"}]}');
$record_examples []= new SchemaExample('
{"type":"record", "name":"foo", "doc":"doc string",
"fields":[{"name":"bar", "type":"int", "order":"bad"}]}
Expand All @@ -413,12 +413,12 @@ protected static function make_examples()
$record_examples []= new SchemaExample(
'{"type":"record","name":"foo","fields":[{"name":"bar","type":["null","string"],"default":null}]}',
true,
'{"type":"record","name":"foo","fields":[{"name":"bar","type":["null","string"],"default":null}]}');
'{"type":"record","name":"foo","fields":[{"name":"bar","type":[{"type":"null"},{"type":"string"}],"default":null}]}');
// Don't lose the "doc" attributes of record fields.
$record_examples []= new SchemaExample(
'{"type":"record","name":"foo","fields":[{"name":"bar","type":["null","string"],"doc":"Bar name."}]}',
true,
'{"type":"record","name":"foo","fields":[{"name":"bar","type":["null","string"],"doc":"Bar name."}]}');
'{"type":"record","name":"foo","fields":[{"name":"bar","type":[{"type":"null"},{"type":"string"}],"doc":"Bar name."}]}');

$primitive_examples []= new SchemaExample(
'{ "type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2 }',
Expand All @@ -437,12 +437,12 @@ protected static function make_examples()
$array_examples []= new SchemaExample(
'{"type": "array", "logicalType": "foo", "items": "string", "foo": "bar"}',
true,
'{"type":"array","items":"string","logicalType":"foo","foo":"bar"}'
'{"type":"array","items":{"type":"string"},"logicalType":"foo","foo":"bar"}'
);
$map_examples []= new SchemaExample(
'{"type": "map", "logicalType": "foo", "values": "long", "foo": "bar"}',
true,
'{"type":"map","values":"long","logicalType":"foo","foo":"bar"}'
'{"type":"map","values":{"type":"long"},"logicalType":"foo","foo":"bar"}'
);
$record_examples []= new SchemaExample(
'{ "type": "record", "name": "foo", "logicalType": "bar", "fields": [], "foo": "bar" }',
Expand Down

0 comments on commit 5936698

Please sign in to comment.