Skip to content

Commit

Permalink
Improve parameter parsing (#583)
Browse files Browse the repository at this point in the history 
* DigiKey: Skip empty values

* Move improved range detection from LCSCProvider to ParameterDTO class
Improve numeric value detection by moving extra info to value_text

* ParameterDTO: Add micro unit prefix

* Bring $value_text2 to a defined state

* ParameterDTO: Don't overwrite $unit if it's not empty

* ParameterDTO: Don't overwrite $unit if it's not empty

* Correct some inaccuacies in comments

* Added tests and fixed certain edge cases in parsing parameters

* Added more tests for parameter parsing

---------

Co-authored-by: Jan Böhmer <[email protected]>
  • Loading branch information
@frank-f @jbtronics
frank-f and jbtronics authored Apr 15, 2024
1 parent fdf64f9 commit 7a6b045
Show file tree
Hide file tree
Showing 4 changed files with 139 additions and 32 deletions.
66 changes: 56 additions & 10 deletions src/Services/InfoProviderSystem/DTOs/ParameterDTO.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,10 @@ public function __construct(

/**
* This function tries to decide on the value, if it is a numerical value (which is then stored in one of the value_*) fields) or a text value (which is stored in value_text).
* It is possible to give ranges like 1...2 here, which will be parsed as value_min: 1.0, value_max: 2.0.
* It is possible to give ranges like 1...2 (or 1~2) here, which will be parsed as value_min: 1.0, value_max: 2.0.
*
* For certain expressions (like ranges) the unit is automatically extracted from the value, if no unit is given
* @TODO Rework that, so that the difference between parseValueField and parseValueIncludingUnit is clearer or merge them
* @param string $name
* @param string|float $value
* @param string|null $unit
Expand All @@ -54,23 +57,66 @@ public function __construct(
*/
public static function parseValueField(string $name, string|float $value, ?string $unit = null, ?string $symbol = null, ?string $group = null): self
{
//If we encounter something like 2.5@text, then put the "@text" into text_value and continue with the number parsing
if (is_string($value) && preg_match('/^(.+)(@.+)$/', $value, $matches) === 1) {
$value = $matches[1];
$value_text = $matches[2];
} else {
$value_text = null;
}

//If the value is just a number, we assume thats the typical value
if (is_float($value) || is_numeric($value)) {
return new self($name, value_typ: (float) $value, unit: $unit, symbol: $symbol, group: $group);
return new self($name, value_text: $value_text, value_typ: (float) $value, unit: $unit, symbol: $symbol,
group: $group);
}

//Try to parse as range
if (str_contains($value, '...')) {
$parts = explode('...', $value);
//If the attribute contains "..." or a tilde we assume it is a range
if (preg_match('/(\.{3}|~)/', $value) === 1) {
$parts = preg_split('/\s*(\.{3}|~)\s*/', $value);
if (count($parts) === 2) {
//Try to extract number and unit from value (allow leading +)
if (empty($unit)) {
[$number, $unit] = self::splitIntoValueAndUnit(ltrim($parts[0], " +")) ?? [$parts[0], null];
} else {
$number = $parts[0];
}

// If the second part has some extra info, we'll save that into value_text
if (!empty($unit) && preg_match('/^(.+' . preg_quote($unit, '/') . ')\s*(.+)$/', $parts[1], $matches) > 0) {
$parts[1] = $matches[1];
$value_text2 = $matches[2];
} else {
$value_text2 = null;
}
[$number2, $unit2] = self::splitIntoValueAndUnit(ltrim($parts[1], " +")) ?? [$parts[1], $unit];

//Ensure that both parts are numerical
if (is_numeric($parts[0]) && is_numeric($parts[1])) {
return new self($name, value_min: (float) $parts[0], value_max: (float) $parts[1], unit: $unit, symbol: $symbol, group: $group);
//If both parts have the same unit and both values are numerical, we'll save it as range
if ($unit === $unit2 && is_numeric($number) && is_numeric($number2)) {
return new self(name: $name, value_text: $value_text2, value_min: (float) $number,
value_max: (float) $number2, unit: $unit, symbol: $symbol, group: $group);
}
}
//If it's a plus/minus value, we'll also treat it as a range
} elseif (str_starts_with($value, '±')) {
[$number, $unit] = self::splitIntoValueAndUnit(ltrim($value, " ±")) ?? [ltrim($value, ' ±'), $unit];
if (is_numeric($number)) {
return new self(name: $name, value_min: -abs((float) $number), value_max: abs((float) $number), unit: $unit, symbol: $symbol, group: $group);
}
}

//If no unit was passed to us, try to extract it from the value
if (empty($unit)) {
[$value, $unit] = self::splitIntoValueAndUnit($value) ?? [$value, null];
}

//Were we successful in trying to reduce the value to a number?
if ($value_text !== null && is_numeric($value)) {
return new self($name, value_text: $value_text, value_typ: (float) $value, unit: $unit, symbol: $symbol,
group: $group);
}

return new self($name, value_text: $value, unit: $unit, symbol: $symbol, group: $group);
return new self($name, value_text: $value.$value_text, unit: $unit, symbol: $symbol, group: $group);
}

/**
Expand Down Expand Up @@ -106,7 +152,7 @@ public static function parseValueIncludingUnit(string $name, string|float $value
*/
public static function splitIntoValueAndUnit(string $value): ?array
{
if (preg_match('/^(?<value>-?[0-9\.]+)\s*(?<unit>[%Ω°℃a-z_\/]+\s?\w{0,4})$/iu', $value, $matches)) {
if (preg_match('/^(?<value>-?[0-9\.]+)\s*(?<unit>[%Ωµ°℃a-z_\/]+\s?\w{0,4})$/iu', $value, $matches)) {
$value = $matches['value'];
$unit = $matches['unit'];

Expand Down
6 changes: 5 additions & 1 deletion src/Services/InfoProviderSystem/Providers/DigikeyProvider.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,10 @@ private function parametersToDTOs(array $parameters, string|null &$footprint_nam
$footprint_name = $parameter['Value'];
}

if (in_array(trim($parameter['Value']), array('', '-'), true)) {
continue;
}

$results[] = ParameterDTO::parseValueIncludingUnit($parameter['Parameter'], $parameter['Value']);
}

Expand Down Expand Up @@ -265,4 +269,4 @@ private function mediaToDTOs(array $media_links): array
];
}

}
}
21 changes: 0 additions & 21 deletions src/Services/InfoProviderSystem/Providers/LCSCProvider.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -323,27 +323,6 @@ private function attributesToParameters(?array $attributes): array
//Skip this attribute if it's empty
if (in_array(trim($attribute['paramValueEn']), array('', '-'), true)) {
continue;
//If the attribute contains a tilde we assume it is a range
} elseif (str_contains($attribute['paramValueEn'], '~')) {
$parts = explode('~', $attribute['paramValueEn']);
if (count($parts) === 2) {
//Try to extract number and unit from value (allow leading +)
[$number, $unit] = ParameterDTO::splitIntoValueAndUnit(ltrim($parts[0], " +")) ?? [$parts[0], null];
[$number2, $unit2] = ParameterDTO::splitIntoValueAndUnit(ltrim($parts[1], " +")) ?? [$parts[1], null];

//If both parts have the same unit and both values are numerical, we assume it is a range
if ($unit === $unit2 && is_numeric($number) && is_numeric($number2)) {
$result[] = new ParameterDTO(name: $attribute['paramNameEn'], value_min: (float) $number, value_max: (float) $number2, unit: $unit, group: null);
continue;
}
}
//If it's a plus/minus value, we'll also it like a range
} elseif (str_starts_with($attribute['paramValueEn'], '±')) {
[$number, $unit] = ParameterDTO::splitIntoValueAndUnit(ltrim($attribute['paramValueEn'], " ±")) ?? [$attribute['paramValueEn'], null];
if (is_numeric($number)) {
$result[] = new ParameterDTO(name: $attribute['paramNameEn'], value_min: -abs((float) $number), value_max: abs((float) $number), unit: $unit, group: null);
continue;
}
}

$result[] = ParameterDTO::parseValueIncludingUnit(name: $attribute['paramNameEn'], value: $attribute['paramValueEn'], group: null);
Expand Down
78 changes: 78 additions & 0 deletions tests/Services/InfoProviderSystem/DTOs/ParameterDTOTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,56 @@ public function parseValueFieldDataProvider(): \Generator
'm',
'test'
];

//Test ranges with tilde
yield [
new ParameterDTO('test', value_min: -1.0, value_max: 2.0, unit: 'kg', symbol: 'm', group: 'test'),
'test',
'-1.0~+2.0', //Leading signs are parsed correctly
'kg',
'm',
'test'
];

//Test ranges with comment
yield [
new ParameterDTO('test', value_text: "Test", value_min: -1.0, value_max: 2.0, unit: 'kg', symbol: 'm',
group: 'test'),
'test',
'-1.0~+2.0 kg Test', //Leading signs are parsed correctly
'kg',
'm',
'test'
];

//Test @comment
yield [
new ParameterDTO('test', value_text: "@comment", value_typ: 1.0, unit: 'kg', symbol: 'm', group: 'test'),
'test',
'1.0@comment',
'kg',
'm',
'test'
];

//Test plus minus range (without unit)
yield [
new ParameterDTO('test', value_min: -1.0, value_max: +1.0, unit: 'kg', symbol: 'm', group: 'test'),
'test',
'±1.0',
'kg',
'm',
'test'
];

yield [ //And with unit
new ParameterDTO('test', value_min: -1.0, value_max: +1.0, unit: 'kg', symbol: 'm', group: 'test'),
'test',
'±1.0kg',
'kg',
'm',
'test'
];
}

public function parseValueIncludingUnitDataProvider(): \Generator
Expand Down Expand Up @@ -142,6 +192,33 @@ public function parseValueIncludingUnitDataProvider(): \Generator
'm',
'test'
];

//Test ranges with tilde
yield [
new ParameterDTO('test', value_min: -1.0, value_max: 2.0, unit: 'kg', symbol: 'm', group: 'test'),
'test',
'-1.0kg~+2.0kg', //Leading signs are parsed correctly
'm',
'test'
];

//Test @comment
yield [
new ParameterDTO('test', value_text: "@comment", value_typ: 1.0, unit: 'kg', symbol: 'm', group: 'test'),
'test',
'1.0 kg@comment',
'm',
'test'
];

//Test plus minus range (without unit)
yield [
new ParameterDTO('test', value_min: -1.0, value_max: +1.0, unit: 'kg', symbol: 'm', group: 'test'),
'test',
'±1.0 kg',
'm',
'test'
];
}

/**
Expand Down Expand Up @@ -175,6 +252,7 @@ public function testSplitIntoValueAndUnit(): void
$this->assertEquals(["70", "℃"], ParameterDTO::splitIntoValueAndUnit("70℃"));

$this->assertEquals(["-5.0", "kg"], ParameterDTO::splitIntoValueAndUnit("-5.0 kg"));
$this->assertEquals(["-5.0", "µg"], ParameterDTO::splitIntoValueAndUnit("-5.0 µg"));

$this->assertNull(ParameterDTO::splitIntoValueAndUnit('kg'));
$this->assertNull(ParameterDTO::splitIntoValueAndUnit('Test'));
Expand Down

0 comments on commit 7a6b045

Please sign in to comment.