Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Embedded Metadata: Add basic LD-JSON support and some missing meta tags #2765

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

Embedded Metadata: Add basic LD-JSON support and some missing meta tags #2765

Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
0585696
EM: Add basic LD-JSON support and some missing meta tags
owcz Dec 31, 2021
14059ec
rmv debugs
owcz Dec 31, 2021
08afdfd
satisfy linter
owcz Dec 31, 2021
2d4f82d
if multiple blocks, get the one where @type = Article or NewsArticle;…
owcz Dec 31, 2021
beabe4b
pull @type = Person if elsewhere in the JSON
owcz Dec 31, 2021
e076529
handle null exception
owcz Jan 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 58 additions & 5 deletions Embedded Metadata.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
"inRepository": true,
"translatorType": 4,
"browserSupport": "gcsibv",
"lastUpdated": "2021-11-07 07:42:50"
"lastUpdated": "2021-12-31 20:04:11"
}

/*
Expand Down Expand Up @@ -646,6 +646,42 @@ function addOtherMetadata(doc, newItem) {
}
}
}

// try LD-JSON for basic fields
// looks like headline/title and url are already handled just fine elsewhere but can add here if we want to override
let ldJSONpayload = text(doc, 'script[type="application/ld+json"]');
if (ldJSONpayload) {
try {
var ldJSON = JSON.parse(ldJSONpayload);
}
catch (e) {}

if (ldJSON) {
Z.debug("Parsing LD-JSON");

if (newItem.creators.length == 0 && ldJSON.author) {
if (ldJSON.author.length > 0) { // for multiple authors
for (let author of ldJSON.author) {
if (author.name != newItem.publicationTitle || author.name.toLowerCase().includes('editor')) { // skip author when written by the site itself
newItem.creators.push(ZU.cleanAuthor(author.name, "author"));
}
}
}
else newItem.creators.push(ZU.cleanAuthor(ldJSON.author.name, 'author')); // single author
}


if (!newItem.date && ldJSON.datePublished) {
newItem.date = ldJSON.datePublished;
}

if (newItem.tags.length == 0 && ldJSON.keywords) {
for (let tag of ldJSON.keywords) {
newItem.tags.push(tag);
}
}
}
}
}

function addLowQualityMetadata(doc, newItem) {
Expand Down Expand Up @@ -688,7 +724,7 @@ function addLowQualityMetadata(doc, newItem) {
}
}
// fall back to "keywords"
if (!newItem.tags.length) {
if (newItem.tags.length == 0) { // this needs to be tested; previously was !...length, which evaluated to true when zero
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this might need to be tested; previously was !newItem.tags.length, which evaluated to true when zero

appeared to work fine on https://www.bloomberg.com/news/articles/2019-09-12/peloton-founder-goes-from-kickstarter-to-a-450-million-fortune

newItem.tags = attr(doc, 'meta[name="keywords" i]', 'content');
}

Expand All @@ -698,6 +734,10 @@ function addLowQualityMetadata(doc, newItem) {
attr(doc, 'meta[name="description" i]', 'content'));
}

if (!newItem.date) {
newItem.date = ZU.xpathText(doc, '//meta[@name="parsely-pub-date"]/@content');
}

if (!newItem.url) {
newItem.url = ZU.xpathText(doc, '//head/link[@rel="canonical"]/@href') || doc.location.href;
}
Expand All @@ -722,7 +762,7 @@ In a worst case scenario, where real authors and social media profiles are mixed
preferable to garbage */
function tryOgAuthors(doc) {
var authors = [];
var ogAuthors = ZU.xpath(doc, '//meta[@property="article:author" or @property="video:director" or @property="music:musician"]');
var ogAuthors = ZU.xpath(doc, '//meta[@property="article:author" or @property="video:director" or @property="music:musician" or @name="parsely-author"]');
for (var i = 0; i < ogAuthors.length; i++) {
if (ogAuthors[i].content && ogAuthors[i].content.search(/(https?:\/\/)?[\da-z.-]+\.[a-z.]{2,6}/) < 0 && ogAuthors[i].content !== "false") {
authors.push(ZU.cleanAuthor(ogAuthors[i].content, "author"));
Expand Down Expand Up @@ -892,9 +932,10 @@ function getAuthorFromByline(doc, newItem) {
*/
function finalDataCleanup(doc, newItem) {
if (typeof newItem.tags == 'string') {
Z.debug("newItem.tags is a string");
newItem.tags = [newItem.tags];
}
if (newItem.tags && newItem.tags.length && Zotero.parentTranslator) {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

not sure what Zotero.parentTranslator was checking for here—I'd think this should run whenever EM is finding a tag string to parse, not only when it's the parent? Unless the point is to only run when it has a child translator so the child translator can choose what to do with the sometimes garbage tags?

if (newItem.tags && newItem.tags.length == 1) {
if (exports.splitTags) {
var tags = [];
for (let i in newItem.tags) {
Expand All @@ -913,11 +954,14 @@ function finalDataCleanup(doc, newItem) {
newItem.tags = tags;
}
}
if (newItem.tags.length > 0) newItem.tags = scrubLowercaseTags(newItem.tags);

/* what else is automatically adding tags? if it's bad data, don't import it, but resetting all tags (per below) kills good LD-JSON tags
else {
// Unless called from another translator, don't include automatic tags,
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What else is automatically adding tags? Resetting all tags here—the default—kills (now) good LD-JSON tags. Better to just avoid anything that adds garbage tags if we want to be vigilant, rather than adding and immediately wiping it out?

// because most of the time they are not right
newItem.tags = [];
}
}*/

// Cleanup DOI
if (newItem.DOI) {
Expand Down Expand Up @@ -984,6 +1028,15 @@ function relativeToAbsolute(doc, url) {
return location.replace(/([^/]\/)[^/]+$/, '$1') + url;
}

function scrubLowercaseTags(tags) {
for (let tag of tags) {
if (tag == tag.toLowerCase()) {
tags[tags.indexOf(tag)] = ZU.capitalizeTitle(tag, true);
}
}
return tags;
}

var exports = {
doWeb: doWeb,
detectWeb: detectWeb,
Expand Down