Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

🎼 Explication du score de rapprochement #235

Closed
wants to merge 2 commits into from
Closed

🎼 Explication du score de rapprochement #235

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a38f313
🎨 Restructure score function to include explain attribute
cristianpb Apr 25, 2021
194c0d9
✨ Add levenshtein score to explain for name
cristianpb Apr 25, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 26 additions & 1 deletion backend/src/models/entities.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,14 +131,39 @@ export interface Modification {
review?: Review;
};

export interface Score {
score?: number;
birthDate?: number
deathDate?: number
name?: number;
sex?: number;
birthLocation?: number;
deathLocation?: number;
}

export interface Explain {
name?: {
particles: boolean;
nameSwap: boolean;
first: {
levenshteinScore: number;
soundexScore: number;
}
last: {
levenshteinScore: number;
soundexScore: number;
}
}
}

export interface Person {
score: number;
source: string;
sourceLine: number;
id: string;
name: Name;
sex: 'M'|'F';
scores: ScoreResult;
scores: Score;
birth: {
date: string;
location: Location;
Expand Down
8 changes: 4 additions & 4 deletions backend/src/score.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@ describe('score.ts - Score function', () => {
}
}
});
expect(score).to.contain.all.keys(['score', 'birthDate', 'birthLocation', 'name'])
expect(score.score).to.equal(0.73);
expect(score.scores).to.contain.all.keys(['score', 'birthDate', 'birthLocation', 'name'])
expect(score.scores.score).to.equal(0.73);
});


Expand Down Expand Up @@ -96,8 +96,8 @@ describe('score.ts - Score function', () => {
}
}
});
expect(score).to.contain.all.keys(['score', 'birthLocation', 'name'])
expect(score.birthLocation).to.contain.all.keys(['score', 'city', 'code'])
expect(score.scores).to.contain.all.keys(['score', 'birthLocation', 'name'])
expect(score.scores.birthLocation).to.contain.all.keys(['score', 'city', 'code'])

});

Expand Down
88 changes: 56 additions & 32 deletions backend/src/score.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { RequestBody } from './models/requestInput';
import { Person, Location, Name, RequestField, ScoreParams } from './models/entities';
import { Person, Location, Name, RequestField, ScoreParams, Score, Explain } from './models/entities';
import { distance } from 'fastest-levenshtein';
import damlev from 'damlev';
import fuzz from 'fuzzball';
Expand Down Expand Up @@ -182,9 +182,10 @@ const firstNameSexMismatch = (firstNameA: string, firstNameB: string): boolean =
return /^.?(e|a)$/.test(firstA.replace(firstB, '')) || /^.?(e|a)$/.test(firstB.replace(firstA, ''));
}

let scoreName = (nameA: Name, nameB: Name, sex: string): any => {
let scoreName = (nameA: Name, nameB: Name, sex: string, explainScore: boolean): any => {
if ((!nameA.first && !nameA.last) || (!nameB.first && !nameB.last)) { return blindNameScore }
let score:any;
let explain:any;
const firstA = firstNameNorm(nameA.first as string|string[]);
const lastA = lastNameNorm(nameA.last as string|string[]);
const firstB = firstNameNorm(nameB.first as string|string[]);
Expand All @@ -197,6 +198,14 @@ let scoreName = (nameA: Name, nameB: Name, sex: string): any => {
let firstFirstA; let firstFirstB; let scoreFirstALastB; let fuzzScore;
const scoreFirst = round(scoreToken(firstA, firstB));
const scoreLast = round(scoreToken(lastA, lastB));
explain = {
first: {
levenshtein: scoreFirst
},
last: {
levenshtein: scoreLast
}
}
score = round(Math.max(
scoreFirst * (scoreLast ** thisLastNamePenalty),
Math.max(
Expand Down Expand Up @@ -261,7 +270,11 @@ let scoreName = (nameA: Name, nameB: Name, sex: string): any => {
score.particleScore = particleScore;
}
}
return score;
if (explainScore) {
return {score, explain};
} else {
return {score};
}
}

const scoreToken = (tokenA: string|string[], tokenB: string|string[], option?: any): number => {
Expand Down Expand Up @@ -631,48 +644,54 @@ scoreSex = timer(scoreSex, 'scoreSex',100);
scoreDate = timer(scoreDate, 'scoreDate',1000);

export class ScoreResult {
score: number;
birthDate?: number
deathDate?: number
name?: number;
sex?: number;
birthLocation?: number;
deathLocation?: number;
scores: Score;
explain: Explain;

constructor(request: RequestBody, result: Person, params: ScoreParams = {}) {
this.scores = {}
const pruneScore = params.pruneScore !== undefined ? params.pruneScore : defaultPruneScore
// TODO: use input parameter
// const explainScore = params.explainScore !== undefined ? params.explainScore : false;
const explainScore = true;
if (explainScore) this.explain = {}
if (request.birthDate) {
this.birthDate = scoreDate(request.birthDate, result.birth.date, params.dateFormat,
this.scores.birthDate = scoreDate(request.birthDate, result.birth.date, params.dateFormat,
result.birth && result.birth.location && result.birth.location.countryCode && (result.birth.location.countryCode !== 'FRA')
);
}
if (request.firstName || request.lastName) {
if ((pruneScore < scoreReduce(this, true)) || !this.birthDate) {
if ((pruneScore < scoreReduce(this.scores, true)) || !this.scores.birthDate) {
if (result.sex && result.sex === 'F') {
if (request.legalName) {
this.name = scoreName({first: request.firstName, last: [request.lastName, request.legalName]}, result.name, 'F');
const scoreNameResult = scoreName({first: request.firstName, last: [request.lastName, request.legalName]}, result.name, 'F', explainScore);
this.scores.name = scoreNameResult.score
if (explainScore) this.explain.name = scoreNameResult.explain
} else {
this.name = scoreName({first: request.firstName, last: request.lastName}, result.name, 'F');
const scoreNameResult = scoreName({first: request.firstName, last: request.lastName}, result.name, 'M', explainScore);
this.scores.name = scoreNameResult.score
if (explainScore) this.explain.name = scoreNameResult.explain
}
} else {
this.name = scoreName({first: request.firstName, last: request.lastName}, result.name, 'M');
const scoreNameResult = scoreName({first: request.firstName, last: request.lastName}, result.name, 'M', explainScore);
this.scores.name = scoreNameResult.score
if (explainScore) this.explain.name = scoreNameResult.explain
}
} else {
this.score = 0
this.scores.score = 0
}
}
if (request.sex) {
if (pruneScore < scoreReduce(this, true)) {
this.sex = scoreSex(request.sex, result.sex);
if (pruneScore < scoreReduce(this.scores, true)) {
this.scores.sex = scoreSex(request.sex, result.sex);
} else {
this.score = 0
this.scores.score = 0
}
} else if (request.firstName && firstNameSexMismatch(request.firstName, result.name.first as string)) {
this.sex = firstNameSexPenalty;
this.scores.sex = firstNameSexPenalty;
}
// birthLocation
if (pruneScore < scoreReduce(this, true)) {
this.birthLocation = scoreLocation({
if (pruneScore < scoreReduce(this.scores, true)) {
this.scores.birthLocation = scoreLocation({
city: request.birthCity,
code: request.birthLocationCode,
departmentCode: request.birthDepartment,
Expand All @@ -681,20 +700,20 @@ export class ScoreResult {
longitude: request.birthGeoPoint?.longitude
}, result.birth.location);
} else {
this.score = 0
this.scores.score = 0
}
if (request.deathDate || request.lastSeenAliveDate) {
if (pruneScore < scoreReduce(this, true)) {
this.deathDate = scoreDate(request.deathDate || `>${request.lastSeenAliveDate}`, result.death.date, params.dateFormat,
if (pruneScore < scoreReduce(this.scores, true)) {
this.scores.deathDate = scoreDate(request.deathDate || `>${request.lastSeenAliveDate}`, result.death.date, params.dateFormat,
result.death && result.death.location && result.death.location.countryCode && (result.death.location.countryCode !== 'FRA')
);
} else {
this.score = 0
this.scores.score = 0
}
}
if ((request.deathCity || request.deathLocationCode || request.deathCountry || request.deathDepartment || request.deathGeoPoint)) {
if (pruneScore < scoreReduce(this, true)) {
this.deathLocation = scoreLocation({
if (pruneScore < scoreReduce(this.scores, true)) {
this.scores.deathLocation = scoreLocation({
city: request.deathLocation,
code: request.deathLocationCode,
departmentCode: request.deathDepartment,
Expand All @@ -703,17 +722,20 @@ export class ScoreResult {
longitude: request.deathGeoPoint?.longitude
}, result.death.location);
} else {
this.score = 0
this.scores.score = 0
}
}
if (!this.score) {
this.score = scoreReduce(this, true)
if (!this.scores.score) {
this.scores.score = scoreReduce(this.scores, true)
}
}
}

export const scoreResults = (request: RequestBody, results: Person[], params: ScoreParams): Person[] => {
const pruneScore = params.pruneScore !== undefined ? params.pruneScore : defaultPruneScore
// TODO: use input parameter
// const explainScore = params.explainScore !== undefined ? params.explainScore : false;
const explainScore = true;
const candidateNumber = params.candidateNumber || 1;
let maxScore = 0;
let perfectScoreNumber = 0;
Expand All @@ -728,7 +750,9 @@ export const scoreResults = (request: RequestBody, results: Person[], params: Sc
.filter((result:any) => result.score > 0)
.map((result:any) => {
try {
result.scores = new ScoreResult(request, result, params);
const scoreResult = new ScoreResult(request, result, params);
result.scores = scoreResult.scores
if (explainScore) result.explain = scoreResult.explain
const perfectScores =
((result.scores.name && result.scores.name.score >= perfectScoreThreshold) ? 1 : 0) +
((result.scores.birtDate && result.scores.birthDate.score === 1) ? 1 : 0) +
Expand Down