Skip to content

Commit

Permalink
Merge pull request #163 from biothings/scoring-overhaul
Browse files Browse the repository at this point in the history 
#634: Scoring overhaul
  • Loading branch information
@tokebe
tokebe authored Aug 8, 2023
2 parents df278d0 + f3ca0aa commit 32eb684
Show file tree
Hide file tree
Showing 5 changed files with 228 additions and 67 deletions.
117 changes: 117 additions & 0 deletions __test__/unittest/score.test.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
const { calculateScore, exportForTesting } = require('../../src/results_assembly/score');
const { record_weight, text_mined_record_weight, ngd_weight, LENGTH_PENALTY, scaled_sigmoid } = exportForTesting;

describe('Test score function', () => {
const ngdPairs = {
'C0678941-C0267841': 0.5,
'C4548369-C0678941': 0.6,
'C4548369-C0267841': 0.7
};

const sampleComboSimple = [
{
inputQNodeID: 'nB',
outputQNodeID: 'nC',
inputPrimaryCuries: new Set(['UMLS:C0678941']),
outputPrimaryCuries: new Set(['MONDO:0006633']),
inputUMLS: new Set(['C0678941']),
outputUMLS: new Set(['C0267841']),
isTextMined: [ true ],
qEdgeID: 'eB',
recordHashes: new Set(['a'])
},
{
inputQNodeID: 'nA',
outputQNodeID: 'nB',
inputPrimaryCuries: new Set(['PUBCHEM.COMPOUND:77843966']),
outputPrimaryCuries: new Set(['UMLS:C0678941']),
inputUMLS: new Set(['C4548369']),
outputUMLS: new Set(['C0678941']),
isTextMined: [ true ],
qEdgeID: 'eA',
recordHashes: new Set(['b'])
}
];

const sampleComboComplex = [
{
inputQNodeID: 'nB',
outputQNodeID: 'nC',
inputPrimaryCuries: new Set(['UMLS:C0678941']),
outputPrimaryCuries: new Set(['MONDO:0006633']),
inputUMLS: new Set(['C0678941']),
outputUMLS: new Set(['C0267841']),
isTextMined: [ true, false, true ],
qEdgeID: 'eB',
recordHashes: new Set(['a', 'b', 'c'])
},
{
inputQNodeID: 'nA',
outputQNodeID: 'nB',
inputPrimaryCuries: new Set(['PUBCHEM.COMPOUND:77843966']),
outputPrimaryCuries: new Set(['UMLS:C0678941']),
inputUMLS: new Set(['C4548369']),
outputUMLS: new Set(['C0678941']),
isTextMined: [ true, true, true ],
qEdgeID: 'eA',
recordHashes: new Set(['b', 'c', 'd'])
},
{
inputQNodeID: 'nA',
outputQNodeID: 'nC',
inputPrimaryCuries: new Set(['PUBCHEM.COMPOUND:77843966']),
outputPrimaryCuries: new Set(['MONDO:0006633']),
inputUMLS: new Set(['C4548369']),
outputUMLS: new Set(['C0267841']),
isTextMined: [ false, false ],
qEdgeID: 'eC',
recordHashes: new Set(['c', 'd'])
}
];

test('Test calculateScore function - simple case w/ ngd', () => {
const eAScore = text_mined_record_weight + ngd_weight * (1 / ngdPairs['C4548369-C0678941']);
const eBScore = text_mined_record_weight + ngd_weight * (1 / ngdPairs['C0678941-C0267841']);
const expected_score = scaled_sigmoid((eBScore + eAScore) / Math.pow(2, LENGTH_PENALTY));

const res = calculateScore(sampleComboSimple, ngdPairs);
expect(res.score).toBe(expected_score);
expect(res.scoredByNGD).toBeTruthy();
});

test('Test calculateScore function - simple case w/o ngd', () => {
const eAScore = text_mined_record_weight;
const eBScore = text_mined_record_weight;
const expected_score = scaled_sigmoid((eBScore + eAScore) / Math.pow(2, LENGTH_PENALTY));

const res = calculateScore(sampleComboSimple, {});
expect(res.score).toBe(expected_score);
expect(res.scoredByNGD).toBeFalsy();
});

test('Test calculateScore function - complex case w/ ngd', () => {
const eAScore = 2 * text_mined_record_weight + 1 * record_weight + ngd_weight * (1 / ngdPairs['C4548369-C0678941']);
const eBScore = 3 * text_mined_record_weight + 0 * record_weight + ngd_weight * (1 / ngdPairs['C0678941-C0267841']);
const eCScore = 0 * text_mined_record_weight + 2 * record_weight + ngd_weight * (1 / ngdPairs['C4548369-C0267841']);

const expected_score = scaled_sigmoid((eBScore + eAScore) / Math.pow(2, LENGTH_PENALTY) + eCScore / Math.pow(1, LENGTH_PENALTY));

const res = calculateScore(sampleComboComplex, ngdPairs);
expect(res.score).toBe(expected_score);
expect(res.scoredByNGD).toBeTruthy();
});

test('Test calculateScore function - complex case w/o ngd', () => {
const eAScore = 2 * text_mined_record_weight + 1 * record_weight;
const eBScore = 3 * text_mined_record_weight + 0 * record_weight;
const eCScore = 0 * text_mined_record_weight + 2 * record_weight;

const expected_score = scaled_sigmoid((eBScore + eAScore) / Math.pow(2, LENGTH_PENALTY) + eCScore / Math.pow(1, LENGTH_PENALTY));

const res = calculateScore(sampleComboComplex, {});
expect(res.score).toBe(expected_score);
expect(res.scoredByNGD).toBeFalsy();
});
});


9 changes: 9 additions & 0 deletions src/config.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,3 +60,12 @@ exports.EDGE_ATTRIBUTES_USED_IN_RECORD_HASH = [
"biolink:log_odds_ratio",
"biolink:total_sample_size",
];

// based on https://github.com/biolink/biolink-model/blob/master/infores_catalog.yaml
exports.text_mining_api_infores = [
'infores:biothings-semmeddb',
'infores:scibite',
'infores:semmeddb',
'infores:text-mining-provider-cooccurrence',
'infores:text-mining-provider-targeted'
];
23 changes: 2 additions & 21 deletions src/inferred_mode/inferred_mode.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ const utils = require('../utils');
const async = require('async');
const biolink = require('../biolink');
const { getTemplates } = require('./template_lookup');
const { addNormalizedScores } = require('../results_assembly/score');
const { scaled_sigmoid, inverse_scaled_sigmoid } = require('../results_assembly/score');

module.exports = class InferredQueryHandler {
constructor(parent, TRAPIQueryHandler, queryGraph, logs, options, path, predicatePath, includeReasoner) {
Expand Down Expand Up @@ -304,7 +304,7 @@ module.exports = class InferredQueryHandler {
const resScore = translatedResult.analyses[0].score;
if (typeof combinedResponse.message.results[resultID].analyses[0].score !== 'undefined') {
combinedResponse.message.results[resultID].analyses[0].score = resScore
? Math.max(combinedResponse.message.results[resultID].analyses[0].score, resScore)
? scaled_sigmoid(inverse_scaled_sigmoid(combinedResponse.message.results[resultID].analyses[0].score) + inverse_scaled_sigmoid(resScore))
: combinedResponse.message.results[resultID].analyses[0].score;
} else {
combinedResponse.message.results[resultID].analyses[0].score = resScore;
Expand Down Expand Up @@ -519,25 +519,6 @@ module.exports = class InferredQueryHandler {
this.parent
.getSummaryLog(combinedResponse, combinedResponse.logs, resultQueries)
.forEach((log) => combinedResponse.logs.push(log));
let scoredResults = 0;
let unscoredResults = 0;
combinedResponse.message.results.forEach((result) => {
const scoreFromEdges = Object.values(result.analyses[0].edge_bindings).reduce((count, qEdge_bindings) => {
return count + qEdge_bindings.length;
}, 0);
if (result.analyses[0].score > scoreFromEdges) {
scoredResults += 1;
} else {
unscoredResults += 1;
}
});
combinedResponse.logs.push(
new LogEntry(
'INFO',
null,
`Scoring Summary: (${scoredResults}) scored / (${unscoredResults}) unscored`,
).getLog(),
);
}
combinedResponse.logs = combinedResponse.logs.map((log) => log.toJSON());

Expand Down
25 changes: 17 additions & 8 deletions src/results_assembly/query_results.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ const LogEntry = require('../log_entry');
const { getScores, calculateScore } = require('./score');
const { Record } = require('@biothings-explorer/api-response-transform');
const { enrichTrapiResultsWithPfocrFigures } = require('./pfocr');
const config = require('../config');

/**
* @type { Record }
Expand Down Expand Up @@ -171,8 +172,11 @@ module.exports = class TrapiResultsAssembler {
outputQNodeID: record.object.qNodeID,
inputPrimaryCurie: record.subject.curie,
outputPrimaryCurie: record.object.curie,
inputUMLS: record.subject.UMLS, //add umls for scoring
outputUMLS: record.object.UMLS, //add umls for scoring
// info for scoring
inputUMLS: record.subject.UMLS,
outputUMLS: record.object.UMLS,
isTextMined: config.text_mining_api_infores.includes(record.apiInforesCurie),
// end info for scoring
qEdgeID: qEdgeID,
recordHash: record.recordHash,
});
Expand Down Expand Up @@ -361,18 +365,23 @@ module.exports = class TrapiResultsAssembler {
const consolidatedSolutionRecord = {
inputQNodeID: solutionRecord_0.inputQNodeID,
outputQNodeID: solutionRecord_0.outputQNodeID,
inputUMLS: solutionRecord_0.inputUMLS,
outputUMLS: solutionRecord_0.outputUMLS,
inputPrimaryCuries: new Set(),
outputPrimaryCuries: new Set(),
inputUMLS: new Set(),
outputUMLS: new Set(),
isTextMined: [],
qEdgeID: solutionRecord_0.qEdgeID,
recordHashes: new Set(),
};
solutionRecords.forEach(
({ inputQNodeID, outputQNodeID, inputPrimaryCurie, outputPrimaryCurie, qEdgeID, recordHash }) => {
//debug(` inputQNodeID: ${inputQNodeID}, inputPrimaryCurie: ${inputPrimaryCurie}, outputQNodeID ${outputQNodeID}, outputPrimaryCurie: ${outputPrimaryCurie}`)
({ inputQNodeID, outputQNodeID, inputPrimaryCurie, outputPrimaryCurie, inputUMLS, outputUMLS, isTextMined, qEdgeID, recordHash }) => {
consolidatedSolutionRecord.inputPrimaryCuries.add(inputPrimaryCurie);
consolidatedSolutionRecord.outputPrimaryCuries.add(outputPrimaryCurie);
consolidatedSolutionRecord.inputUMLS.add(...inputUMLS);
consolidatedSolutionRecord.outputUMLS.add(...outputUMLS);
if (!consolidatedSolutionRecord.recordHashes.has(recordHash)) {
consolidatedSolutionRecord.isTextMined.push(isTextMined);
}
consolidatedSolutionRecord.recordHashes.add(recordHash);
},
);
Expand Down Expand Up @@ -443,12 +452,12 @@ module.exports = class TrapiResultsAssembler {
debug('Error enriching with PFOCR figures: ', err);
this.logs.push(new LogEntry('DEBUG', null, 'Error enriching with PFOCR figures: ', err).getLog());
}
debug(`Successfully scored ${resultsWithScore} results, couldn't score ${resultsWithoutScore} results.`);
debug(`Scored ${resultsWithScore} results with NGD score, scored ${resultsWithoutScore} results without NGD.`);
this.logs.push(
new LogEntry(
'DEBUG',
null,
`Successfully scored ${resultsWithScore} results, couldn't score ${resultsWithoutScore} results.`,
`Scored ${resultsWithScore} results with NGD score, scored ${resultsWithoutScore} results without NGD.`,
{
type: 'scoring',
scored: resultsWithScore,
Expand Down
Loading

0 comments on commit 32eb684

Please sign in to comment.