From caab3d265ddbb3bcf74b0666f553d4dae1de90cb Mon Sep 17 00:00:00 2001 From: zoe-translates <116055375+zoe-translates@users.noreply.github.com> Date: Fri, 31 Mar 2023 15:16:01 +0000 Subject: [PATCH 001/125] NASA ADS: Handle the ampersand character (&) in BibCode. (#2998) --- NASA ADS.js | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 66 insertions(+), 3 deletions(-) diff --git a/NASA ADS.js b/NASA ADS.js index 11db8fb0f0b..eff67edfff4 100644 --- a/NASA ADS.js +++ b/NASA ADS.js @@ -1,7 +1,7 @@ { "translatorID": "7987b420-e8cb-4bea-8ef7-61c2377cd686", "label": "NASA ADS", - "creator": "Tim Hostetler and Abe Jellinek", + "creator": "Tim Hostetler, Abe Jellinek, and Zoë C. Ma", "target": "^https://ui\\.adsabs\\.harvard\\.edu/(search|abs)/", "minVersion": "3.0", "maxVersion": "", @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2021-08-12 05:31:17" + "lastUpdated": "2023-03-20 08:38:43" } /* @@ -53,7 +53,7 @@ function getSearchResults(doc) { } function extractId(url) { - return /\/abs\/([^/]+)/.exec(url)[1]; + return decodeURIComponent(/\/abs\/([^/]+)/.exec(url)[1]); } function getTypeFromId(id) { @@ -393,6 +393,69 @@ var testCases = [ "seeAlso": [] } ] + }, + { + "type": "web", + "url": "https://ui.adsabs.harvard.edu/abs/2023A%26ARv..31....1A/abstract", + "items": [ + { + "itemType": "journalArticle", + "title": "Origin of the elements", + "creators": [ + { + "lastName": "Arcones", + "firstName": "Almudena", + "creatorType": "author" + }, + { + "lastName": "Thielemann", + "firstName": "Friedrich-Karl", + "creatorType": "author" + } + ], + "date": "2023-12-01", + "DOI": "10.1007/s00159-022-00146-x", + "ISSN": "0935-4956", + "abstractNote": "What is the origin of the oxygen we breathe, the hydrogen and oxygen (in form of water H2O) in rivers and oceans, the carbon in all organic compounds, the silicon in electronic hardware, the calcium in our bones, the iron in steel, silver and gold in jewels, the rare earths utilized, e.g. in magnets or lasers, lead or lithium in batteries, and also of naturally occurring uranium and plutonium? The answer lies in the skies. Astrophysical environments from the Big Bang to stars and stellar explosions are the cauldrons where all these elements are made. The papers by Burbidge (Rev Mod Phys 29:547-650, 1957) and Cameron (Publ Astron Soc Pac 69:201, 1957), as well as precursors by Bethe, von Weizsäcker, Hoyle, Gamow, and Suess and Urey provided a very basic understanding of the nucleosynthesis processes responsible for their production, combined with nuclear physics input and required environment conditions such as temperature, density and the overall neutron/proton ratio in seed material. Since then a steady stream of nuclear experiments and nuclear structure theory, astrophysical models of the early universe as well as stars and stellar explosions in single and binary stellar systems has led to a deeper understanding. This involved improvements in stellar models, the composition of stellar wind ejecta, the mechanism of core-collapse supernovae as final fate of massive stars, and the transition (as a function of initial stellar mass) from core-collapse supernovae to hypernovae and long duration gamma-ray bursts (accompanied by the formation of a black hole) in case of single star progenitors. Binary stellar systems give rise to nova explosions, X-ray bursts, type Ia supernovae, neutron star, and neutron star-black hole mergers. All of these events (possibly with the exception of X-ray bursts) eject material with an abundance composition unique to the specific event and lead over time to the evolution of elemental (and isotopic) abundances in the galactic gas and their imprint on the next generation of stars. In the present review, we want to give a modern overview of the nucleosynthesis processes involved, their astrophysical sites, and their impact on the evolution of galaxies.", + "extra": "ADS Bibcode: 2023A&ARv..31....1A", + "libraryCatalog": "NASA ADS", + "pages": "1", + "publicationTitle": "Astronomy and Astrophysics Review", + "url": "https://ui.adsabs.harvard.edu/abs/2023A&ARv..31....1A", + "volume": "31", + "attachments": [ + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [ + { + "tag": "Big Bang nucleosynthesis" + }, + { + "tag": "Compact binary mergers" + }, + { + "tag": "Core collapse" + }, + { + "tag": "Element abundance" + }, + { + "tag": "Galactic evolution" + }, + { + "tag": "Stellar evolution" + }, + { + "tag": "Supernovae" + } + ], + "notes": [], + "seeAlso": [] + } + ] } ] /** END TEST CASES **/ From 623d3da4571fec01cdab0caad60c996eecda3806 Mon Sep 17 00:00:00 2001 From: vinothk-hw <78914673+vinothk-hw@users.noreply.github.com> Date: Fri, 31 Mar 2023 21:09:34 +0530 Subject: [PATCH 002/125] Add translator for Access Engineering (#2809) --- Access Engineering.js | 333 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 333 insertions(+) create mode 100644 Access Engineering.js diff --git a/Access Engineering.js b/Access Engineering.js new file mode 100644 index 00000000000..b1c05abce30 --- /dev/null +++ b/Access Engineering.js @@ -0,0 +1,333 @@ +{ + "translatorID": "d120a8a7-9d45-446e-8c18-ad9ef0a6bf47", + "label": "Access Engineering", + "creator": "Vinoth K - highwirepress.com", + "target": "^https?://www\\.accessengineeringlibrary\\.com/content/(book|chapter|case-study|video|calculator|tutorial)", + "minVersion": "3.0", + "maxVersion": "", + "priority": 100, + "inRepository": true, + "translatorType": 4, + "browserSupport": "gcsibv", + "lastUpdated": "2023-03-07 08:48:13" +} + +/* + ***** BEGIN LICENSE BLOCK ***** + Copyright © 2020-2021 Vinoth K - highwirepress.com + + This file is part of Zotero. + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + ***** END LICENSE BLOCK ***** +*/ + +function detectWeb(doc, url) { + let title = attr(doc, 'meta[name="citation_title"]', 'content'); + if (title) { + if (doc.querySelector('meta[name="citation_isbn"]')) { + let bookTitle = attr(doc, 'meta[name="citation_book_title"]', 'content'); + if (!bookTitle || title == bookTitle) { + return "book"; + } + else { + return "bookSection"; + } + } + else if (url.includes('content/video/')) { + return 'videoRecording'; + } + else { + return "journalArticle"; + } + } + else if (getSearchResults(doc, true)) { + return "multiple"; + } + return false; +} + +function getSearchResults(doc, checkOnly) { + var items = {}; + var found = false; + var rows = doc.querySelectorAll('.results-item a[href]'); + for (let row of rows) { + let href = row.href; + let title = ZU.trimInternal(row.textContent); + if (!href || !title) continue; + if (checkOnly) return true; + found = true; + items[href] = title; + } + return found ? items : false; +} + +function doWeb(doc, url) { + if (detectWeb(doc, url) == "multiple") { + Zotero.selectItems(getSearchResults(doc, false), function (items) { + if (items) ZU.processDocuments(Object.keys(items), scrape); + }); + } + else { + scrape(doc, url); + } +} + +function scrape(doc, url) { + // Missing editions for books and books chapter page + // Removed html element in abstract for video and tutorial page + // Author not updating in metatag correctly from data and read + // client advised to split and handled through custom data attr/obj + // - so we'll fill those in manually. + var translator = Zotero.loadTranslator('web'); + // Embedded Metadata + translator.setTranslator('951c027d-74ac-47d4-a107-9c3069ab7b48'); + translator.setDocument(doc); + translator.setHandler('itemDone', function (obj, item) { + // Edition + let edition = ZU.xpathText(doc, '//meta[@name="citation_edition"]/@content'); + if (edition) item.edition = edition; + + // Author + let author = ZU.xpath(doc, '//ul[@class="contributor-list"]//li//a'); + if (author.length > 0) { + // Handled using data attribute + for (let i = 0; i < author.length; i++) { + item.creators[i].firstName = author[i].getAttribute('data-firstnames'); + item.creators[i].lastName = author[i].getAttribute('data-surname'); + } + } + + // Abstract + let abstractNote = ZU.xpathText(doc, '//meta[@name="citation_abstract"]/@content'); + if (abstractNote) item.abstractNote = ZU.cleanTags(abstractNote); + + item.complete(); + }); + + translator.getTranslatorObject(function (trans) { + // Detect web not get trigger for scape EM translator + // - so wll fill those in manually. + if (detectWeb(doc, url)) { + trans.itemType = detectWeb(doc, url); + } + trans.addCustomFields({ + citation_book_title: "bookTitle" + }); + trans.doWeb(doc, url); + }); +} + +/** BEGIN TEST CASES **/ +var testCases = [ + { + "type": "web", + "url": "https://www.accessengineeringlibrary.com/content/book/9781259860386/", + "items": [ + { + "itemType": "book", + "title": "3D Printer Projects for Makerspaces", + "creators": [ + { + "firstName": "Lydia Sloan", + "lastName": "Cline", + "creatorType": "author" + } + ], + "date": "2017", + "ISBN": "9781259860386", + "abstractNote": "Learn to model and print 3D designs—no experience required!This easy-to-follow guide features twenty 3D printing projects for makers of all skill levels to enjoy. Written in a tutorial, step-by-step manner, 3D Printer Projects for Makerspaces shows how to use Fusion 360, SketchUp, Meshmixer, Remake, and Inkscape to create fun and useful things. Scanning, slicers, silicone molds, settings, and build plate orientation are also covered, as well as post-processing methods that will make your prints really pop!Inside, you9ll learn to model, analyze, and print a:• Phone case• Coin bank• Art stencil• Cookie cutter• Cookie dunker• Personalized key fob• Lens cap holder• Lithophane night-light• Pencil cup with applied sketch• Business card with QR code• Bronze pendant• Soap mold• Hanging lampshade• Scanned Buddha charm• And more!", + "edition": "1st Edition", + "language": "en", + "libraryCatalog": "www.accessengineeringlibrary.com", + "publisher": "McGraw-Hill Education", + "url": "https://www.accessengineeringlibrary.com/content/book/9781259860386", + "attachments": [ + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.accessengineeringlibrary.com/content/book/9781259860386/chapter/chapter12", + "items": [ + { + "itemType": "bookSection", + "title": "PROJECT 12: Lithophane Night-Light", + "creators": [ + { + "firstName": "Lydia Sloan", + "lastName": "Cline", + "creatorType": "author" + } + ], + "date": "2017", + "ISBN": "9781259860386", + "abstractNote": "Learn to model and print 3D designs—no experience required!This easy-to-follow guide features twenty 3D printing projects for makers of all skill levels to enjoy. Written in a tutorial, step-by-step manner, 3D Printer Projects for Makerspaces shows how to use Fusion 360, SketchUp, Meshmixer, Remake, and Inkscape to create fun and useful things. Scanning, slicers, silicone molds, settings, and build plate orientation are also covered, as well as post-processing methods that will make your prints really pop!Inside, you'll learn to model, analyze, and print a:• Phone case• Coin bank• Art stencil• Cookie cutter• Cookie dunker• Personalized key fob• Lens cap holder• Lithophane night-light• Pencil cup with applied sketch• Business card with QR code• Bronze pendant• Soap mold• Hanging lampshade• Scanned Buddha charm• And more!", + "bookTitle": "3D Printer Projects for Makerspaces", + "edition": "1st Edition", + "language": "en", + "libraryCatalog": "www.accessengineeringlibrary.com", + "publisher": "McGraw-Hill Education", + "shortTitle": "PROJECT 12", + "url": "https://www.accessengineeringlibrary.com/content/book/9781259860386/chapter/chapter12", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.accessengineeringlibrary.com/content/video/V4768153299001", + "items": [ + { + "itemType": "videoRecording", + "title": "10% Infill and a Bridge", + "creators": [ + { + "firstName": "Lydia", + "lastName": "Cline", + "creatorType": "author" + } + ], + "date": "2016", + "abstractNote": "This video shows an item being printed with a 10% infill and includes a bridge.", + "language": "en", + "libraryCatalog": "www.accessengineeringlibrary.com", + "studio": "McGraw-Hill Education", + "url": "https://www.accessengineeringlibrary.com/content/video/V4768153299001", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.accessengineeringlibrary.com/content/calculator/S0018_Analysis_of_AC_and_DC_Circuits_Basic_Calculations", + "items": [ + { + "itemType": "journalArticle", + "title": "Analysis of A.C. and D.C. Circuits - Basic Calculations", + "creators": [ + { + "firstName": "William", + "lastName": "Prudhomme", + "creatorType": "author" + } + ], + "date": "2018/12/13/", + "abstractNote": "Software simulation programs are generally used for modeling and designing complex electronic circuits and applications, but frequently only a basic calculation is needed to solve an immediate design problem or to calculate the value of a specific circuit element. This Excel workbook addresses this need by automating the calculation of over 70 basic electronics formulas in direct current (d.c.) and alternating current (a.c.) circuits and applications.", + "language": "en", + "libraryCatalog": "www.accessengineeringlibrary.com", + "url": "https://www.accessengineeringlibrary.com/content/calculator/S0018_Analysis_of_AC_and_DC_Circuits_Basic_Calculations", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.accessengineeringlibrary.com/content/case-study/CS0004_Atrial_Fibrillation", + "items": [ + { + "itemType": "journalArticle", + "title": "Atrial Fibrillation: Improving Therapy via Engineering Advancements", + "creators": [ + { + "firstName": "Michael J.", + "lastName": "Rust", + "creatorType": "author" + } + ], + "date": "2020-04-23", + "abstractNote": "This case will explore atrial fibrillation from several perspectives, including the underlying physiology, clinical relevance, and instrumentation used for diagnosis and therapy. Students will identify and investigate unmet clinical needs that led to recent developments in technologies to treat atrial fibrillation.", + "language": "en", + "libraryCatalog": "www.accessengineeringlibrary.com", + "shortTitle": "Atrial Fibrillation", + "url": "https://www.accessengineeringlibrary.com/content/case-study/CS0004_Atrial_Fibrillation", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.accessengineeringlibrary.com/content/tutorial/T0002_Open_Channel_Flow_Calculations_with_the_Manning_Equation", + "items": [ + { + "itemType": "journalArticle", + "title": "Open Channel Flow Calculations with the Manning Equation using Excel Spreadsheets", + "creators": [ + { + "firstName": "Harlan", + "lastName": "H. Bengtson", + "creatorType": "author" + } + ], + "date": "2014-02-01", + "abstractNote": "This tutorial teaches the Manning equation and its use for uniform open channel flow calculations, including the hydraulic radius, Manning roughness coefficient, and normal depth. There are example problems and illustrations show how to use spreadsheets for the calculations.", + "language": "en", + "libraryCatalog": "www.accessengineeringlibrary.com", + "url": "https://www.accessengineeringlibrary.com/content/tutorial/T0002_Open_Channel_Flow_Calculations_with_the_Manning_Equation", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.accessengineeringlibrary.com/search?query=&f%5B0%5D=content_type%3ABooks&f%5B1%5D=book_component%3ATitles", + "items": "multiple" + } +] +/** END TEST CASES **/ From 1ab84b67f6b9e70f30f381916cce839ae4321e6b Mon Sep 17 00:00:00 2001 From: Maun Suang Boey Date: Sat, 1 Apr 2023 02:51:58 +1100 Subject: [PATCH 003/125] Financial Times: Fix creators, section, URL (#3006) --- Financial Times.js | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/Financial Times.js b/Financial Times.js index 14e6b06a2cb..e44aff27400 100644 --- a/Financial Times.js +++ b/Financial Times.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2021-05-27 16:17:23" + "lastUpdated": "2023-03-27 05:16:52" } /* @@ -73,6 +73,9 @@ function doWeb(doc, url) { } function scrape(_doc, url) { + // amp.ft.com/ now redirects to www.ft.com/, and there appears + // to be no other way to get AMP versions of Financial Times articles, + // but we'll keep the current code structure for now, just in case. ZU.processDocuments(url.replace('www.ft.com/', 'amp.ft.com/'), scrapeAmp); } @@ -81,7 +84,8 @@ function scrapeAmp(doc, url) { let meta = [...doc.querySelectorAll('script[type="application/ld+json"]')] .map(elem => JSON.parse(elem.textContent)) - .find(json => json['@type'] != 'WebSite'); + .find(json => json['@type'] != 'WebSite' + && json['@type'] != 'BreadcrumbList'); if (!meta) { throw new Error("No article metadata (probably hit paywall)"); } @@ -90,14 +94,12 @@ function scrapeAmp(doc, url) { item.date = ZU.strToISO(meta.datePublished); item.abstractNote = meta.description || text('.article-standfirst'); - // something funky is going on with the JSON-LD authors, so we'll just - // parse from the HTML - item.creators = [...doc.querySelectorAll('a.article-author-byline__author')] - .map(link => ZU.cleanAuthor(link.innerText, 'author', false)); + item.creators = meta.author + .map(obj => ZU.cleanAuthor(obj.name, 'author', false)); item.publicationTitle = 'Financial Times'; - item.section = text('h2.primary-brand a') - || text('h2.primary-theme a'); - item.url = meta.mainEntityofPage; + item.section = text('a[data-trackable="primary-brand"]') + || text('a[data-trackable="primary-theme"]'); + item.url = attr('link[rel="canonical"]', 'href'); item.libraryCatalog = ''; item.attachments.push({ title: "Snapshot", From 258c5610b42d56fa3588a2c5f2e83a9a064bbed1 Mon Sep 17 00:00:00 2001 From: Dan Stillman Date: Fri, 31 Mar 2023 18:03:46 -0400 Subject: [PATCH 004/125] IMDb: Fix saving after site change Probably fixes #2999 -- I'd guess this change just rolled out first for non-English pages --- IMDb.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IMDb.js b/IMDb.js index c6a88bdd418..37b42c01979 100644 --- a/IMDb.js +++ b/IMDb.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2023-03-07 11:44:52" + "lastUpdated": "2023-03-31 22:02:24" } /* @@ -97,7 +97,7 @@ function scrape(doc, _url) { } item.title = title; // note that json only has the original title - var transTitle = ZU.trimInternal(ZU.xpathText(doc, "//h1/text()")); + var transTitle = ZU.trimInternal(ZU.xpathText(doc, "//h1//text()")); if (transTitle && transTitle !== item.title) addExtra(item, "Translated title: " + transTitle); item.programTitle = doc.title.match(/(?:"([^"]+)")?/)[1]; From a55c0fecbd2259956c7fff34f347116479e52b0a Mon Sep 17 00:00:00 2001 From: Dan Stillman Date: Fri, 31 Mar 2023 18:06:17 -0400 Subject: [PATCH 005/125] IMDb: Update tests --- IMDb.js | 66 ++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 20 deletions(-) diff --git a/IMDb.js b/IMDb.js index 37b42c01979..19d679794c1 100644 --- a/IMDb.js +++ b/IMDb.js @@ -182,42 +182,42 @@ var testCases = [ { "firstName": "Norma", "lastName": "Aleandro", - "creatorType": "castMember" + "creatorType": "contributor" }, { "firstName": "Héctor", "lastName": "Alterio", - "creatorType": "castMember" + "creatorType": "contributor" }, { "firstName": "Chunchuna", "lastName": "Villafañe", - "creatorType": "castMember" + "creatorType": "contributor" } ], "date": "1985-11-08", "abstractNote": "During the final months of Argentinian Military Dictatorship in 1983, a high school teacher sets out to find out who the mother of her adopted daughter is.", - "distributor": "Historias Cinematograficas Cinemania, Progress Communications", + "distributor": "Historias Cinematograficas, Progress Communications", "extra": "Translated title: The Official Story\nIMDb ID: tt0089276\nevent-location: Argentina", - "genre": "Drama, History, War", + "genre": "Drama, History", "libraryCatalog": "IMDb", "runningTime": "1h52m", "attachments": [], "tags": [ { - "tag": "adopted daughter" + "tag": "bigotry" }, { - "tag": "high school teacher" + "tag": "military" }, { - "tag": "lawyer" + "tag": "military junta" }, { - "tag": "school" + "tag": "teacher" }, { - "tag": "thumb sucking" + "tag": "torture victim" } ], "notes": [], @@ -256,21 +256,21 @@ var testCases = [ { "firstName": "Eero", "lastName": "Melasniemi", - "creatorType": "castMember" + "creatorType": "contributor" }, { "firstName": "Kristiina", "lastName": "Halkola", - "creatorType": "castMember" + "creatorType": "contributor" }, { "firstName": "Pekka", "lastName": "Autiovuori", - "creatorType": "castMember" + "creatorType": "contributor" } ], "date": "1966-10-21", - "abstractNote": "Depiction of four urban youths and their excursion to the countryside.", + "abstractNote": "Two student couples go camping in the Finnish countryside; partner swapping and interpersonal dynamics - with a touch of their philosophy - between them all arise.", "distributor": "FJ-Filmi", "extra": "IMDb ID: tt0060613\nevent-location: Finland", "genre": "Drama", @@ -285,13 +285,13 @@ var testCases = [ "tag": "dance" }, { - "tag": "film star" + "tag": "female topless nudity" }, { - "tag": "snakebite" + "tag": "film star" }, { - "tag": "topless" + "tag": "snakebite" } ], "notes": [], @@ -315,17 +315,43 @@ var testCases = [ { "firstName": "David", "lastName": "Attenborough", - "creatorType": "castMember" + "creatorType": "contributor" + }, + { + "firstName": "Pete", + "lastName": "McCowen", + "creatorType": "contributor" + }, + { + "firstName": "Jerome", + "lastName": "Poncet", + "creatorType": "contributor" } ], "date": "2017-02-18", "abstractNote": "Wildlife documentary series with David Attenborough, beginning with a look at the remote islands which offer sanctuary to some of the planet's rarest creatures.", - "episodeNumber": "S1 E1", "extra": "IMDb ID: tt6142646\nevent-location: United Kingdom", "libraryCatalog": "IMDb", "programTitle": "Planet Earth II", + "runningTime": "51m", "attachments": [], - "tags": [], + "tags": [ + { + "tag": "documentary episode" + }, + { + "tag": "earth" + }, + { + "tag": "impossible" + }, + { + "tag": "impressed" + }, + { + "tag": "planet" + } + ], "notes": [], "seeAlso": [] } From 2371723baea3f6ebf093297d57c4459b49cb1f53 Mon Sep 17 00:00:00 2001 From: swifterslb Date: Mon, 3 Apr 2023 16:28:00 +0200 Subject: [PATCH 006/125] FAO Publications: Update for new layout (#2975) --- FAO Publications.js | 904 ++++++++++++++++++++++++++++---------------- 1 file changed, 586 insertions(+), 318 deletions(-) diff --git a/FAO Publications.js b/FAO Publications.js index 75c5249e24e..b5e7a456865 100644 --- a/FAO Publications.js +++ b/FAO Publications.js @@ -2,14 +2,14 @@ "translatorID": "4883f662-29df-44ad-959e-27c9d036d165", "label": "FAO Publications", "creator": "Bin Liu ", - "target": "^https?://www\\.fao\\.org/(documents|publications)/", - "minVersion": "3.0", + "target": "^https?://www\\.fao\\.org/(publications|documents)/", + "minVersion": "5.0", "maxVersion": "", "priority": 100, "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2021-08-31 04:00:00" + "lastUpdated": "2023-04-01 16:44:37" } /* @@ -30,11 +30,17 @@ */ function detectWeb(doc, url) { // Just differentiate single and multiple. - // Identify item type (book or conferencePaper) based on "fdr_label" class. - if (url.includes('card')) { + if (url.includes('/card/')) { let isConferencePaper = false; let confMetaName = ['اسم الاجتماع', '会议名称', 'Meeting Name', 'Nom de la réunion', 'Название мероприятия', 'Nombre de la reunión']; - let labelArray = doc.querySelectorAll('.fdr_label'); + let labelArray = []; + if (url.includes('/publications/')) { + labelArray = doc.querySelectorAll('.fdr_label'); // Identify item type (book or conferencePaper) based on "fdr_label" class. + } + else if (url.includes('/documents/')) { + labelArray = doc.querySelectorAll('.fw-bold'); // Identify item type (book or conferencePaper) based on "fw-bold" class. + // Page layout for meeting documents is not functioning properly at "documents" pages (e.g. https://www.fao.org/documents/card/en/c/ND423EN/ and http://www.fao.org/documents/card/zh/c/mw246ZH/ ). Keep the code for now because it doesn't interfere with books and meeting documents are very few. + } for (let i = 0; i < labelArray.length; i++) { for (let j = 0; j < confMetaName.length; j++) { isConferencePaper = labelArray[i].innerText.includes(confMetaName[j]); @@ -62,21 +68,85 @@ function detectWeb(doc, url) { return false; } -function cleanMeta(str) { - // clean meta fields obtained from page +function cleanMetaPub(str) { + // clean meta fields obtained from page for "publications" pages if (str.includes(';') === false) { return str.slice(str.indexOf(':') + 2); } else { - var strArray = str.slice(str.indexOf(':') + 2).split(';'); + let strArray = str.slice(str.indexOf(':') + 2).split(';'); + return strArray; + } +} + +function cleanMetaDoc(str) { + // clean meta fields obtained from page for "documents" pages + if (str.includes(';') === false) { + return str; + } + else { + let strArray = str.split(';').filter(String); // split by semicolon and remove empty elements return strArray; } } +function getLang(str) { + // language: 2 or 3 letters following ISO 639 + // indicated by the last 1-3 letters in PDF file name (langCode) + // One good example is the various language versions of http://www.fao.org/publications/card/en/c/I2801E + let langCode, lang = ''; + let matches = str.match(/([a-z]+)\.pdf$/i); + if (matches) { + langCode = matches[1]; + } + // In the new PDF naming scheme, langCode follows ISO 639. + if (langCode.length > 1) { + lang = langCode.toLowerCase(); + } + // In the old PDF naming scheme, langCode is one lower/upper case letter and only differentiates between the 6 UN languages. + else if ((langCode == 'a') || (langCode == 'A')) { + lang = 'ar'; + } + else if ((langCode == 'c') || (langCode == 'C')) { + lang = 'zh'; + } + else if ((langCode == 'e') || (langCode == 'E')) { + lang = 'en'; + } + else if ((langCode == 'f') || (langCode == 'F')) { + lang = 'fr'; + } + else if ((langCode == 'r') || (langCode == 'R')) { + lang = 'ru'; + } + else if ((langCode == 's') || (langCode == 'S')) { + lang = 'es'; + } + else { // Other languages are usually designated 'o'. Using 'else' just to be safe. + lang = 'other'; + } + return lang; +} + function scrape(doc, url) { var newItem = new Z.Item(); + var abs, existingMeta = {}; + var textVariable = { // declarations for metadata names as appeared in document pages in different languages + date: ['سنة النشر', '出版年份', 'Year of publication', 'Année de publication', 'Год издания', 'Fecha de publicación'], + publisher: ['الناشر', '出版方', 'Publisher', 'Éditeur', 'Издатель', 'Editor'], + place: ['مكان النشر', '出版地点', 'Place of publication', 'Lieu de publication', 'Место публикации', 'Lugar de publicacion'], + pages: ['الصفحات', '页数', 'Pages', 'Страницы', 'Páginas'], + ISBN: ['الرقم الدولي الموحد للكتاب', 'ISBN'], + author: ['الكاتب', '作者', 'Author', 'Auteur', 'Автор', 'Autor'], + seriesTitle: ['العنوان التسلسي', '系列标题', 'Serial Title', 'Titre de la série', 'Название серии', 'Título de la serie'], + seriesNumber: ['رقم المسلسل', '系列号码', 'Series number', 'Numéro de série', 'Серийный номер', 'Número de serie'], + conference: ['اسم الاجتماع', '会议名称', 'Meeting Name', 'Nom de la réunion', 'Название мероприятия', 'Nombre de la reunión'] + }; + var metaText = []; + var DOIMatch, pdfUrl, mainTitle, subTitle, metaResult, conferenceWeb = ''; + var DOILead = 'https://doi.org/'; - if (url.includes('card')) { + if (url.includes('/card/')) { // attach document card URL and snapshot // TEMP: Disable at least until we have post-JS snapshots /* newItem.attachments.push({ @@ -85,235 +155,365 @@ function scrape(doc, url) { mimeType: 'text/html', snapshot: true }); */ + if (url.includes('/publications/')) { + //* ********* Begin fixed-location variables ********** - //* ********* Begin fixed-location variables ********** - - // Some variables always appear and appear at the same location in all document pages. + // Some variables always appear and appear at the same location in all document pages. - // abstract - var abs = doc.getElementById("mainContentN0"); - // The childrens of `abs` are the label "Abstract:" in a strong-tag, - // the abstract in several p-tags or text nodes directly, and possibly - // a note about other languages which begins also with a strong-tag. - if (abs) { - var children = abs.childNodes; - var abstractFound = false; - for (let child of children) { - if (child.tagName == "STRONG" || (child.nodeType == 1 && ZU.xpathText(child, './/strong'))) { - if (abstractFound) { - break; // stop when another strong tag is found + // abstract + abs = doc.getElementById("mainContentN0"); + // The childrens of `abs` are the label "Abstract:" in a strong-tag, + // the abstract in several p-tags or text nodes directly, and possibly + // a note about other languages which begins also with a strong-tag. + if (abs) { + let children = abs.childNodes; + let abstractFound = false; + for (let child of children) { + if (child.tagName == "STRONG" || (child.nodeType == Node.ELEMENT_NODE && text(child, 'strong'))) { + if (abstractFound) { + break; // stop when another strong tag is found + } + else { + abstractFound = true; + continue; // exclude the label "Abstract" + } } - else { - abstractFound = true; - continue; // exclude the label "Abstract" + if (newItem.abstractNote) { + if (newItem.abstractNote.slice(-1) !== "\n") { + newItem.abstractNote += "\n\n"; + } + newItem.abstractNote += child.textContent; } - } - if (newItem.abstractNote) { - if (newItem.abstractNote.slice(-1) !== "\n") { - newItem.abstractNote += "\n\n"; + else { + newItem.abstractNote = child.textContent; } - newItem.abstractNote += child.textContent; } - else { - newItem.abstractNote = child.textContent; + // DOI: Some docs contain DOI as a separate paragraph in abs field + if (abs.innerText.includes(DOILead)) { + DOIMatch = abs.innerText.match(/https:\/\/doi\.org\/(.+)/i); + newItem.DOI = DOIMatch[1]; } } - // DOI: Some docs contain DOI as a separate paragraph in abs field - var DOILead = 'https://doi.org/'; - if (abs.innerText.includes(DOILead)) { - var DOIMatch = abs.innerText.match(/https:\/\/doi\.org\/(.+)/i); - newItem.DOI = DOIMatch[1]; + + // attach PDF: PDF link in innerHTML of "dynafef_det" class. + pdfUrl = attr(doc, '.dynafef_det a[href$=".pdf"]', 'href'); + newItem.attachments.push({ + url: pdfUrl, + title: 'Full Text PDF', + mimeType: 'application/pdf' + }); + + // url + newItem.url = url; + + //language + newItem.language = getLang(pdfUrl); + + // title: use colon to connect main title and subtitle (if subtitle exists) + mainTitle = text(doc, '#headerN0 > h1'); + subTitle = text(doc, 'h4.csc-firstHeader'); + if (!subTitle) { + newItem.title = mainTitle; + } + else if ((newItem.language == 'zh') || (newItem.language == 'ja')) { + newItem.title = mainTitle + ':' + subTitle; + } + else { + newItem.title = mainTitle + ': ' + subTitle; } - } - // attach PDF - var pdfUrl = ZU.xpath(doc, '//*[@id="mainRightN0"]/div[2]/a')[0].href; - newItem.attachments.push({ - url: pdfUrl, - title: 'Full Text PDF', - mimeType: 'application/pdf' - }); - // url - newItem.url = url; - // language: 2 or 3 letters following ISO 639 - // indicated by the last 1-3 letters in PDF file name (langCode) - // One good example is the various language versions of http://www.fao.org/publications/card/en/c/I2801E - var langCode = ''; - var matches = pdfUrl.match(/([a-z]+)\.pdf$/i); - if (matches) { - langCode = matches[1]; - } - // In the new PDF naming scheme, langCode follows ISO 639. - if (langCode.length > 1) { - newItem.language = langCode.toLowerCase(); - } - // In the old PDF naming scheme, langCode is one lower/upper case letter and only differentiates between the 6 UN languages. - else if ((langCode == 'a') || (langCode == 'A')) { - newItem.language = 'ar'; - } - else if ((langCode == 'c') || (langCode == 'C')) { - newItem.language = 'zh'; - } - else if ((langCode == 'e') || (langCode == 'E')) { - newItem.language = 'en'; - } - else if ((langCode == 'f') || (langCode == 'F')) { - newItem.language = 'fr'; - } - else if ((langCode == 'r') || (langCode == 'R')) { - newItem.language = 'ru'; - } - else if ((langCode == 's') || (langCode == 'S')) { - newItem.language = 'es'; - } - else { // Other languages are usually designated 'o'. Using 'else' just to be safe. - newItem.language = 'other'; - } - // title: use colon to connect main title and subtitle (if subtitle exists) - var mainTitle = ZU.xpathText(doc, '//*[@id="headerN0"]/h1'); - var subTitle = ZU.xpathText(doc, '//h4[@class="csc-firstHeader h1"]'); - if (!subTitle) { - newItem.title = mainTitle; - } - else if ((newItem.language == 'zh') || (newItem.language == 'ja')) { - newItem.title = mainTitle + ':' + subTitle; - } - else { - newItem.title = mainTitle + ': ' + subTitle; - } - //* ********* End fixed-location variables ********** - - - //* ********* Begin dynamic-location variables ********** - - // Variables that appear neither in all document pages nor at same positions in the pages. - var metaText = ZU.xpath(doc, '//*[@id="mainN0"]')[0].innerText.split('\n'); // scrape text of meta area and split into an array based on line breaks. - // get what variables are listed in the page, save to object existingMeta - var textVariable = { // declarations for metadata names as appeared in document pages in different languages - date: ['سنة النشر', '出版年份', 'Year of publication', 'Année de publication', 'Год издания', 'Fecha de publicación'], - publisher: ['الناشر', '出版方', 'Publisher', 'Éditeur', 'Издатель', 'Editor'], - place: ['مكان النشر', '出版地点', 'Place of publication', 'Lieu de publication', 'Место публикации', 'Lugar de publicacion'], - pages: ['الصفحات', '页数', 'Pages', 'Страницы', 'Páginas'], - ISBN: ['الرقم الدولي الموحد للكتاب', 'ISBN'], - author: ['الكاتب', '作者', 'Author', 'Auteur', 'Автор', 'Autor'], - seriesTitle: ['العنوان التسلسي', '系列标题', 'Serial Title', 'Titre de la série', 'Название серии', 'Título de la serie'], - seriesNumber: ['رقم المسلسل', '系列号码', 'Series number', 'Numéro de série', 'Серийный номер', 'Número de serie'], - conference: ['اسم الاجتماع', '会议名称', 'Meeting Name', 'Nom de la réunion', 'Название мероприятия', 'Nombre de la reunión'], - tags: ['المعجم الكلمات الموضوع', 'AGROVOC', 'Agrovoc', 'АГРОВОК'] - }; - var existingMeta = {}; - for (let i = 0; i < metaText.length; i++) { - for (let key in textVariable) { - for (let j = 0; j < textVariable[key].length; j++) { - if (metaText[i].includes(textVariable[key][j])) { - existingMeta[key] = metaText[i]; + //* ********* End fixed-location variables ********** + + + //* ********* Begin dynamic-location variables ********** + + // Variables that appear neither in all document pages nor at same positions in the pages. + // scrape text of meta area and split into an array based on line breaks. + metaText = text(doc, '#fdr_label').split('\n'); + // get what variables are listed in the page, save to object existingMeta + for (let i = 0; i < metaText.length; i++) { + for (let key in textVariable) { + for (let j = 0; j < textVariable[key].length; j++) { + if (metaText[i].includes(textVariable[key][j])) { + existingMeta[key] = metaText[i]; + } } } } + + for (let key in existingMeta) { + metaResult = cleanMetaPub(existingMeta[key]); + + // date + if (key.includes('date')) { + newItem.date = metaResult; + } + // publisher + if (key.includes('publisher')) { + newItem.publisher = metaResult; + } + // place + if (key.includes('place')) { + newItem.place = metaResult; + } + // number of pages + if (key.includes('pages')) { + newItem.numPages = metaResult.match(/\d+/)[0]; + } + // ISBN + if (key.includes('ISBN')) { + newItem.ISBN = ZU.cleanISBN(metaResult, false); + } + // author(s): whether there is one or more authors; whether last and first name are separated by ',' (if not, use single-field mode). + if (key.includes('author')) { + if (Array.isArray(metaResult)) { // If there are more than 1 authors, metaResult returns an array. + for (let i = 0; i < metaResult.length; i++) { + if (metaResult[i].includes(',')) { + newItem.creators.push(ZU.cleanAuthor(metaResult[i], 'author', true)); + } + else { + newItem.creators.push({ + lastName: metaResult[i], + creatorType: 'author', + fieldMode: 1 + }); + } + } + } + else if (metaResult.includes(',')) { + newItem.creators.push(ZU.cleanAuthor(metaResult, 'author', true)); + } + else { + newItem.creators.push({ + lastName: metaResult, + creatorType: 'author', + fieldMode: 1 + }); + } + } + // tag (Agrovoc) + if (key.includes('tags')) { + for (var i = 0; i < metaResult.length; i++) { + newItem.tags[i] = metaResult[i].trim(); + } + } + // seriesTitle + if (key.includes('seriesTitle')) { + newItem.series = metaResult; + } + // seriesNumber + if (key.includes('seriesNumber')) { + newItem.seriesNumber = metaResult; + } + // conferenceName: save for later conditions. + if (key.includes('conference')) { + conferenceWeb = metaResult[0]; + newItem.conferenceName = conferenceWeb; + } + } + + // If there's no publisher, use 'FAO' as publisher. + if (!newItem.publisher) { + newItem.publisher = 'FAO'; + } + // If there's no place, use 'Rome, Italy' as place. + if (!newItem.place) { + newItem.place = 'Rome, Italy'; + } + // If there's no author, use 'FAO' as author. + if (!newItem.creators.length) { + newItem.creators.push({ + lastName: 'FAO', + creatorType: 'author', + fieldMode: 1 + }); + } + // If conference exists in document page, the itemType is 'conferencePaper'; otherwise it's 'book'. + if (conferenceWeb) { + newItem.itemType = 'conferencePaper'; + } + else { + newItem.itemType = 'book'; + } + //* ********* End dynamic-location variables ********** } + if (url.includes('documents')) { + //* ********* Begin fixed-location variables ********** + + // Some variables always appear and appear at the same location in all document pages. - for (let key in existingMeta) { - var metaResult = cleanMeta(existingMeta[key]); + // abstract + abs = doc.getElementsByClassName("_card-body-info-center")[0]; + // abstractNote should be all text before the class "others-info". See example: https://www.fao.org/documents/card/en/c/ca8466en + var otherInfo = abs.querySelectorAll(".others-info")[0]; + var keywords = abs.querySelectorAll(".tags-list")[0]; // "KEYWORDS:" + tags + newItem.abstractNote = (abs.innerText.replace(otherInfo.innerText, '').replace(keywords.innerText, '')).trim(); - // date - if (key.includes('date')) { - newItem.date = metaResult; + // tags: class="badge" within abs + var tags = abs.querySelectorAll(".badge"); + for (let i = 0; i < tags.length; i++) { + newItem.tags[i] = tags[i].innerText.trim(); } - // publisher - if (key.includes('publisher')) { - newItem.publisher = metaResult; + + // attach PDF: PDF link in innerHTML of "_card-buttons-downloads" class. + pdfUrl = (doc.getElementsByClassName("_card-buttons-downloads")[0].innerHTML).match(/http\S*\.pdf/gi)[0]; + newItem.attachments.push({ + url: pdfUrl, + title: 'Full Text PDF', + mimeType: 'application/pdf' + }); + + // url + newItem.url = url; + + // language: 2 or 3 letters following ISO 639 + newItem.language = getLang(pdfUrl); + + // title: use colon to connect main title and subtitle (if subtitle exists) + mainTitle = doc.getElementsByClassName("page-title")[0].innerText; + var subTitleElement = doc.getElementsByClassName("sub-title"); + if (subTitleElement.length == '0') { // If there's no sub-title class in the web page, subTitleElement is an empty HTMLCollection with “0” (string, not number) as the length attribute. + newItem.title = mainTitle; } - // place - if (key.includes('place')) { - newItem.place = metaResult; + else if ((newItem.language == 'zh') || (newItem.language == 'ja')) { + newItem.title = mainTitle + ':' + subTitleElement[0].innerText; } - // number of pages - if (key.includes('pages')) { - newItem.numPages = metaResult.match(/\d+/)[0]; + else { + newItem.title = mainTitle + ': ' + subTitleElement[0].innerText; } - // ISBN - if (key.includes('ISBN')) { - newItem.ISBN = ZU.cleanISBN(metaResult, false); + + //* ********* End fixed-location variables ********** + + + //* ********* Begin dynamic-location variables ********** + + // Variables that appear neither in all document pages nor at same positions in the pages. + metaText = doc.getElementsByClassName("_card-body-info-left")[0].innerText; + + // DOI + if (metaText.includes(DOILead)) { + DOIMatch = metaText.match(/https:\/\/doi\.org\/(.+)/i); + newItem.DOI = DOIMatch[1]; } - // author(s): whether there is one or more authors; whether last and first name are separated by ',' (if not, use single-field mode). - if (key.includes('author')) { - if (Array.isArray(metaResult)) { // If there are more than 1 authors, metaResult returns an array. - for (let i = 0; i < metaResult.length; i++) { - if (metaResult[i].includes(',')) { - newItem.creators.push(ZU.cleanAuthor(metaResult[i], 'author', true)); + + // scrape text of meta area and split into an array based on line breaks. + var metaTextArr = metaText.split('\n'); + // get what variables are listed in the page, save to object existingMeta + for (let i = 0; i < metaTextArr.length; i++) { + for (let key in textVariable) { + for (let j = 0; j < textVariable[key].length; j++) { + if (metaTextArr[i].includes(textVariable[key][j])) { + existingMeta[key] = metaTextArr[i + 1]; // In metaTextArr, the value of a meta field always appears at the next element of the meta. } - else { - newItem.creators.push({ - lastName: metaResult[i], - creatorType: 'author', - fieldMode: 1 - }); + } + } + } + + for (let key in existingMeta) { + metaResult = cleanMetaDoc(existingMeta[key]); + + // date + if (key.includes('date')) { + newItem.date = metaResult; + } + // publisher + if (key.includes('publisher')) { + if (Array.isArray(metaResult)) { // differentiate between multiple (array) and single (string) + newItem.publisher = metaResult.join(', '); + } + else { + newItem.publisher = metaResult; + } + } + // place + if (key.includes('place')) { // differentiate between multiple (array) and single (string) + if (Array.isArray(metaResult)) { + newItem.publisher = metaResult.join(', '); + } + else { + newItem.publisher = metaResult; + } + } + // number of pages + if (key.includes('pages')) { + newItem.numPages = metaResult.match(/\d+/)[0]; + } + // ISBN + if (key.includes('ISBN')) { + newItem.ISBN = ZU.cleanISBN(metaResult, false); + } + // author(s): whether there is one or more authors; whether last and first name are separated by ',' (if not, use single-field mode). + if (key.includes('author')) { + if (Array.isArray(metaResult)) { // If there are more than 1 authors, metaResult returns an array. + for (let i = 0; i < metaResult.length; i++) { + if (metaResult[i].includes(',')) { + newItem.creators.push(ZU.cleanAuthor(metaResult[i], 'author', true)); + } + else { + newItem.creators.push({ + lastName: metaResult[i], + creatorType: 'author', + fieldMode: 1 + }); + } } } + else if (metaResult.includes(',')) { + newItem.creators.push(ZU.cleanAuthor(metaResult, 'author', true)); + } + else { + newItem.creators.push({ + lastName: metaResult, + creatorType: 'author', + fieldMode: 1 + }); + } } - else if (metaResult.includes(',')) { - newItem.creators.push(ZU.cleanAuthor(metaResult, 'author', true)); + // seriesTitle + if (key.includes('seriesTitle')) { + newItem.series = metaResult; } - else { - newItem.creators.push({ - lastName: metaResult, - creatorType: 'author', - fieldMode: 1 - }); + // seriesNumber + if (key.includes('seriesNumber')) { + newItem.seriesNumber = metaResult; } - } - // tag (Agrovoc) - if (key.includes('tags')) { - for (var i = 0; i < metaResult.length; i++) { - newItem.tags[i] = metaResult[i].trim(); + // conferenceName + if (key.includes('conference')) { + newItem.conferenceName = metaResult[0]; } } - // seriesTitle - if (key.includes('seriesTitle')) { - newItem.series = metaResult; + // If there's no publisher, use 'FAO' as publisher. + if (!newItem.publisher) { + newItem.publisher = 'FAO'; } - // seriesNumber: extract the number. - if (key.includes('seriesNumber')) { - newItem.seriesNumber = (metaResult.match(/\d+/) || [])[0]; + // If there's no place, use 'Rome, Italy' as place. + if (!newItem.place) { + newItem.place = 'Rome, Italy'; } - // conferenceName: save for later conditions. - if (key.includes('conference')) { - var conferenceWeb = metaResult[0]; - newItem.conferenceName = conferenceWeb; + // If there's no author, use 'FAO' as author. + if (!newItem.creators.length) { + newItem.creators.push({ + lastName: 'FAO', + creatorType: 'author', + fieldMode: 1 + }); } + // If conference exists in document page, the itemType is 'conferencePaper'; otherwise it's 'book'. + if (newItem.conferenceName) { + newItem.itemType = 'conferencePaper'; + } + else { + newItem.itemType = 'book'; + } + //* ********* End dynamic-location variables ********** } - - // If there's no publisher, use 'FAO' as publisher. - if (!newItem.publisher) { - newItem.publisher = 'FAO'; - } - // If there's no place, use 'Rome, Italy' as place. - if (!newItem.place) { - newItem.place = 'Rome, Italy'; - } - // If there's no author, use 'FAO' as author. - if (!newItem.creators.length) { - newItem.creators.push({ - lastName: 'FAO', - creatorType: 'author', - fieldMode: 1 - }); - } - // If conference exists in document page, the itemType is 'conferencePaper'; otherwise it's 'book'. - if (conferenceWeb) { - newItem.itemType = 'conferencePaper'; - } - else { - newItem.itemType = 'book'; - } - //* ********* End dynamic-location variables ********** } newItem.complete(); } - // get items from a multiple-item page -function getSearchResults(doc, checkOnly) { +// Multiple-item searching is no longer provided. +/*function getSearchResults(doc, checkOnly) { var items = {}; var found = false; var rows = ZU.xpath(doc, '//*[@class="item-image"]'); @@ -326,24 +526,23 @@ function getSearchResults(doc, checkOnly) { items[href] = title; } return found ? items : false; -} +}*/ function doWeb(doc, url) { - if (detectWeb(doc, url) == "multiple") { - Z.selectItems(getSearchResults(doc, false), function (items) { - if (!items) { - return; - } - var articles = []; - for (var i in items) { - articles.push(i); - } - ZU.processDocuments(articles, scrape); - }); - } - else { - scrape(doc, url); - } + // if (detectWeb(doc, url) == "multiple") { + // Z.selectItems(getSearchResults(doc, false), function (items) { + // if (!items) { + // return; + // } + // var articles = []; + // for (var i in items) {// articles.push(i); + // } + // ZU.processDocuments(articles, scrape); + // }); + // } + // else { + scrape(doc, url); + // } } // Note on test cases: Because the pages use dynamic elements (which is also why the translator doesn't work for multiple item pages), automatic test in Scaffold doesn't work. Every time a test is needed, use "New Web" to manually add it. @@ -352,12 +551,12 @@ function doWeb(doc, url) { var testCases = [ { "type": "web", - "url": "http://www.fao.org/documents/card/en/c/ca8466en", + "url": "https://www.fao.org/documents/card/en?details=cc0461en", "defer": true, "items": [ { "itemType": "book", - "title": "Responding to the impact of the COVID-19 outbreak on food value chains through efficient logistics", + "title": "The State of World Fisheries and Aquaculture 2022: Towards Blue Transformation", "creators": [ { "lastName": "FAO", @@ -365,15 +564,18 @@ var testCases = [ "fieldMode": 1 } ], - "date": "2020", - "ISBN": "9789251323717", - "abstractNote": "Measures implemented around the world to contain the COVID-19 pandemic have entailed a severe reduction not only in the transportation of goods and services that rely on transport, but also in the migration of labour domestically and internationally. Workers are less available reflecting both disruptions in transportation systems and restrictions to stop the transmission of the disease, within and across borders. \n\nThe Food and Agriculture Organization of the United Nations (FAO) urges countries to maintain functioning food value chains to avoid food shortages, following practices that are being proven to work. This note summarizes some practices that could be useful for governments and the private sector to maintain critical logistical elements in food value chain.\n\nRevised 26 April 2020.\n\nSee the full list of policy briefs related to COVID-19\n\n.", + "date": "2022", + "ISBN": "9789251363645", + "abstractNote": "The 2022 edition of The State of World Fisheries and Aquaculture coincides with the launch of the Decade of Action to deliver the Global Goals, the United Nations Decade of Ocean Science for Sustainable Development and the United Nations Decade on Ecosystem Restoration. It presents how these and other equally important United Nations events, such as the International Year of Artisanal Fisheries and Aquaculture (IYAFA 2022), are being integrated and supported through Blue Transformation, a priority area of FAO’s new Strategic Framework 2022–2031 designed to accelerate achievement of the 2030 Agenda for Sustainable Development in food and agriculture.\n\nThe concept of Blue Transformation emerged from the Thirty-fourth Session of the FAO Committee on Fisheries in February 2021, and in particular the Declaration for Sustainable Fisheries and Aquaculture, which was negotiated and endorsed by all FAO Members. The Declaration calls for support for “an evolving and positive vision for fisheries and aquaculture in the twenty first century, where the sector is fully recognized for its contribution to fighting poverty, hunger and malnutrition.” In this context, Part 1 of this edition of The State of World Fisheries and Aquaculture reviews the world status of fisheries and aquaculture, while Parts 2 and 3 are devoted to Blue Transformation and its pillars on intensifying and expanding aquaculture, improving fisheries management and innovating fisheries and aquaculture value chains. Blue Transformation emphasizes the need for forward-looking and bold actions to be launched or accelerated in coming years to achieve the objectives of the Declaration and in support of the 2030 Agenda. Part 4 covers current and high-impact emerging issues – COVID-19, climate change and gender equality – that require thorough consideration for transformative steps and preparedness to secure sustainable, efficient and equitable fisheries and aquaculture, and finally draws some outlook on future trends based on projections.\n\nThe State of World Fisheries and Aquaculture aims to provide objective, reliable and up-to-date information to a wide audience – policymakers, managers, scientists, stakeholders and indeed everyone interested in the fisheries and aquaculture sector.", "language": "en", "libraryCatalog": "FAO Publications", - "numPages": "4", + "numPages": "266", "place": "Rome, Italy", "publisher": "FAO", - "url": "http://www.fao.org/documents/card/en/c/ca8466en", + "series": "The State of World Fisheries and Aquaculture (SOFIA)", + "seriesNumber": "2022", + "shortTitle": "The State of World Fisheries and Aquaculture 2022", + "url": "https://www.fao.org/documents/card/en?details=cc0461en", "attachments": [ { "title": "Full Text PDF", @@ -382,16 +584,28 @@ var testCases = [ ], "tags": [ { - "tag": "Coronavirus" + "tag": "aquaculture production" }, { - "tag": "agrifood sector" + "tag": "climate change adaptation" }, { - "tag": "infectious diseases" + "tag": "fish trade" }, { - "tag": "logistics" + "tag": "fishery management" + }, + { + "tag": "fishery production" + }, + { + "tag": "fishery resources" + }, + { + "tag": "gender equality" + }, + { + "tag": "sustainable fisheries" }, { "tag": "value chains" @@ -404,35 +618,95 @@ var testCases = [ }, { "type": "web", - "url": "http://www.fao.org/documents/card/en/c/ca8751en/", + "url": "https://www.fao.org/publications/card/en?details=cc0461en", "defer": true, "items": [ { "itemType": "book", - "title": "Blockchain application in seafood value chains", + "title": "The State of World Fisheries and Aquaculture 2022: Towards Blue Transformation", "creators": [ { - "firstName": "F.", - "lastName": "Blaha", - "creatorType": "author" + "lastName": "FAO", + "creatorType": "author", + "fieldMode": 1 + } + ], + "date": "2022", + "ISBN": "9789251363645", + "abstractNote": "The 2022 edition of The State of World Fisheries and Aquaculture coincides with the launch of the Decade of Action to deliver the Global Goals, the United Nations Decade of Ocean Science for Sustainable Development and the United Nations Decade on Ecosystem Restoration. It presents how these and other equally important United Nations events, such as the International Year of Artisanal Fisheries and Aquaculture (IYAFA 2022), are being integrated and supported through Blue Transformation, a priority area of FAO’s new Strategic Framework 2022–2031 designed to accelerate achievement of the 2030 Agenda for Sustainable Development in food and agriculture. \n\nThe concept of Blue Transformation emerged from the Thirty-fourth Session of the FAO Committee on Fisheries in February 2021, and in particular the Declaration for Sustainable Fisheries and Aquaculture, which was negotiated and endorsed by all FAO Members. The Declaration calls for support for “an evolving and positive vision for fisheries and aquaculture in the twenty first century, where the sector is fully recognized for its contribution to fighting poverty, hunger and malnutrition.” In this context, Part 1 of this edition of The State of World Fisheries and Aquaculture reviews the world status of fisheries and aquaculture, while Parts 2 and 3 are devoted to Blue Transformation and its pillars on intensifying and expanding aquaculture, improving fisheries management and innovating fisheries and aquaculture value chains. Blue Transformation emphasizes the need for forward-looking and bold actions to be launched or accelerated in coming years to achieve the objectives of the Declaration and in support of the 2030 Agenda. Part 4 covers current and high-impact emerging issues – COVID-19, climate change and gender equality – that require thorough consideration for transformative steps and preparedness to secure sustainable, efficient and equitable fisheries and aquaculture, and finally draws some outlook on future trends based on projections. \n\nThe State of World Fisheries and Aquaculture aims to provide objective, reliable and up-to-date information to a wide audience – policymakers, managers, scientists, stakeholders and indeed everyone interested in the fisheries and aquaculture sector.\n\nThe following complementary information is available:\n\nRead online the full digital reportSee the interactive storyRead the In Brief\n\nHelp us improve your reading experience\n\nLast updated date 19/08/2022", + "language": "other", + "libraryCatalog": "FAO Publications", + "numPages": "266", + "place": "Rome, Italy", + "publisher": "FAO", + "series": "The State of World Fisheries and Aquaculture (SOFIA)", + "seriesNumber": "2022", + "shortTitle": "The State of World Fisheries and Aquaculture 2022", + "url": "https://www.fao.org/publications/card/en?details=cc0461en", + "attachments": [ + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [ + { + "tag": "aquaculture production" }, { - "firstName": "K.", - "lastName": "Katafono", - "creatorType": "author" + "tag": "climate change adaptation" + }, + { + "tag": "fish trade" + }, + { + "tag": "fishery management" + }, + { + "tag": "fishery production" + }, + { + "tag": "fishery resources" + }, + { + "tag": "gender equality" + }, + { + "tag": "sustainable fisheries" + }, + { + "tag": "value chains" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.fao.org/documents/card/en/c/ca8466en", + "defer": true, + "items": [ + { + "itemType": "book", + "title": "Responding to the impact of the COVID-19 outbreak on food value chains through efficient logistics", + "creators": [ + { + "lastName": "FAO", + "creatorType": "author", + "fieldMode": 1 } ], "date": "2020", - "ISBN": "9789251324530", - "abstractNote": "Innovation through information and communication technologies is a key enabler in transforming food systems and holds great potential to achieve the Sustainable Development Goals. Recent developments, such as mobile technologies, smart networks, drones, remote-sensing, distributed computing, as well as disruptive technologies, such as blockchain, the Internet of things and artificial intelligence, are serving as the premise for a “digital revolution” whereby management of resources can potentially be highly optimized, intelligent and anticipatory. This publication establishes chain traceability as the substrate over which digital solutions need to operate. It provides a comprehensive introduction to blockchain, and covers smart contracts, explores how they relate to blockchain with an example of their use in seafood value chains, and then examines major development and operational considerations for blockchain applications. The publication also analyses the seafood supply chain with considerations on flag, coastal, port, processing and market States. It identifies general control elements (critical tracking events and corresponding key data elements) that form the basis for traceability monitoring and acquisition, and summarizes suitability for blockchain. It also investigates considerations for legality, transparency, species fraud and food safety.", + "ISBN": "9789251323717", + "abstractNote": "Measures implemented around the world to contain the COVID-19 pandemic have entailed a severe reduction not only in the transportation of goods and services that rely on transport, but also in the migration of labour domestically and internationally. Workers are less available reflecting both disruptions in transportation systems and restrictions to stop the transmission of the disease, within and across borders.\n\nThe Food and Agriculture Organization of the United Nations (FAO) urges countries to maintain functioning food value chains to avoid food shortages, following practices that are being proven to work. This note summarizes some practices that could be useful for governments and the private sector to maintain critical logistical elements in food value chain.", "language": "en", "libraryCatalog": "FAO Publications", - "numPages": "56", + "numPages": "4", "place": "Rome, Italy", "publisher": "FAO", - "series": "FAO Fisheries and Aquaculture Circular", - "seriesNumber": "1207", - "url": "http://www.fao.org/documents/card/en/c/ca8751en/", + "url": "https://www.fao.org/documents/card/en/c/ca8466en", "attachments": [ { "title": "Full Text PDF", @@ -441,22 +715,19 @@ var testCases = [ ], "tags": [ { - "tag": "analysis" - }, - { - "tag": "blockchain technology" + "tag": "Coronavirus" }, { - "tag": "fisheries" + "tag": "agrifood sector" }, { - "tag": "food production" + "tag": "infectious diseases" }, { - "tag": "food systems" + "tag": "logistics" }, { - "tag": "traceability" + "tag": "value chains" } ], "notes": [], @@ -466,33 +737,35 @@ var testCases = [ }, { "type": "web", - "url": "http://www.fao.org/documents/card/en/c/I9069EN", + "url": "https://www.fao.org/documents/card/en/c/ca8751en/", "defer": true, "items": [ { "itemType": "book", - "title": "Republic of Moldova Value Chain Gap Analysis", + "title": "Blockchain application in seafood value chains", "creators": [ { - "firstName": "J.", - "lastName": "O'Connell", + "firstName": "F.", + "lastName": "Blaha", "creatorType": "author" }, { - "firstName": "P.", - "lastName": "Kiparisov", + "firstName": "K.", + "lastName": "Katafono", "creatorType": "author" } ], - "date": "2018", - "ISBN": "9789251304839", - "abstractNote": "Agriculture and food industry sectors have a major importance for the Moldovan economy. The Republic of Moldova has one of the highest share of rural population among the countries in Europe and Central Asia, and its agriculture sector significantly contributes to the country’s gross domestic product.\n\nThis work is a part of a series of studies on the value chain development gaps and the environment for doing business for farmers. The goal of this study is to try to consolidate the information on countrywide value chain development gathered from various open sources and based on materials developed in a field mission by FAO officers with an emphasis on the plum and berry value chains. The authors did not aim at close examination of the selected value chains; rather, this paper is a general overview that will be a reference point for future field work in the country.\n\nTo get the results, the authors analysed the legislative history related to value chains, collected materials and statistics from open sources, conducted a field mission and interviewed stakeholders.\n\nThe first part of the report observes the overall situation in the Republic of Moldova with a focus on the agriculture sector, reviewing related legislation, the environment for doing business for farmers, and trade. The paper examines existing support measures for agriculture and covers the banking sector and trade policy. The second part examines value chain actors and overviews the selected value chains of plums and berries. The final part provides recommendations.", + "date": "2020", + "ISBN": "9789251324530", + "abstractNote": "Innovation through information and communication technologies is a key enabler in transforming food systems and holds great potential to achieve the Sustainable Development Goals. Recent developments, such as mobile technologies, smart networks, drones, remote-sensing, distributed computing, as well as disruptive technologies, such as blockchain, the Internet of things and artificial intelligence, are serving as the premise for a “digital revolution” whereby management of resources can potentially be highly optimized, intelligent and anticipatory. This publication establishes chain traceability as the substrate over which digital solutions need to operate. It provides a comprehensive introduction to blockchain, and covers smart contracts, explores how they relate to blockchain with an example of their use in seafood value chains, and then examines major development and operational considerations for blockchain applications. The publication also analyses the seafood supply chain with considerations on flag, coastal, port, processing and market States. It identifies general control elements (critical tracking events and corresponding key data elements) that form the basis for traceability monitoring and acquisition, and summarizes suitability for blockchain. It also investigates considerations for legality, transparency, species fraud and food safety.", "language": "en", "libraryCatalog": "FAO Publications", - "numPages": "65", - "place": "Budapest, Hungary", + "numPages": "56", + "place": "Rome, Italy", "publisher": "FAO", - "url": "http://www.fao.org/documents/card/en/c/I9069EN", + "series": "FAO Fisheries and Aquaculture Circular", + "seriesNumber": "No. 1207", + "url": "https://www.fao.org/documents/card/en/c/ca8751en/", "attachments": [ { "title": "Full Text PDF", @@ -501,28 +774,22 @@ var testCases = [ ], "tags": [ { - "tag": "Republic of Moldova" - }, - { - "tag": "agricultural sector" - }, - { - "tag": "data analysis" + "tag": "analysis" }, { - "tag": "economic analysis" + "tag": "blockchain technology" }, { - "tag": "economic infrastructure" + "tag": "fisheries" }, { - "tag": "economic situation" + "tag": "food production" }, { - "tag": "research" + "tag": "food systems" }, { - "tag": "supply chain" + "tag": "traceability" } ], "notes": [], @@ -532,7 +799,7 @@ var testCases = [ }, { "type": "web", - "url": "http://www.fao.org/documents/card/en/c/ca7988en/", + "url": "https://www.fao.org/documents/card/en/c/ca7988en/", "defer": true, "items": [ { @@ -554,7 +821,7 @@ var testCases = [ "place": "Rome, Italy", "publisher": "FAO", "shortTitle": "FAO publications catalogue 2020", - "url": "http://www.fao.org/documents/card/en/c/ca7988en/", + "url": "https://www.fao.org/documents/card/en/c/ca7988en/", "attachments": [ { "title": "Full Text PDF", @@ -582,7 +849,7 @@ var testCases = [ }, { "type": "web", - "url": "http://www.fao.org/publications/card/fr/c/77dbd058-8dd4-4295-af77-23f6b28cc683/", + "url": "https://www.fao.org/documents/card/fr/c/77dbd058-8dd4-4295-af77-23f6b28cc683/", "defer": true, "items": [ { @@ -602,15 +869,15 @@ var testCases = [ ], "date": "2016", "ISBN": "9789252094890", - "abstractNote": "Ce livre nous emmène au cœur des zones de forêts denses et sahéliennes de l’Afrique centrale, un écosystème précieux et essentiel à la vie quotidienne de ses habitants, représentant l’un des trois principaux ensembles boisés tropicaux de la planète. Dix pays (Burundi, Cameroun, Congo, Gabon, Guinée Equatoriale, République Centrafricaine, République Démocratique du Congo, Rwanda, Sao Tomé & Principe, Tchad) abritent ces forêts et savanes, riches d’importantes ressources naturelles. Ils ont en com mun une longue histoire liée à la colonisation, suivie d'une expérience de coopération multiforme depuis les indépendances qui évolue incontestablement vers une intégration économique et monétaire. De nos jours, alors que les équilibres séculaires entre l’homme et la nature semblent ébranlés, que la sécurité alimentaire, la lutte contre la pauvreté et la préservation de la biodiversité et des ressources forestières sont devenus des enjeux mondiaux ; à l’heure où la croissance démographique non m aîtrisée fragilise le maintien des écosystèmes forestiers tout en accentuant les conflits liés à la recherche d’espace vital, le phénomène des changements climatiques vient davantage sonder le génie créateur des populations forestières dans la préservation et la gestion durable de la forêt et des produits forestiers non ligneux (PFNL) qui en sont issus. Cette publication est l’œuvre du personnel technique de la FAO, avec la contribution des partenaires internationaux et locaux engagés dans l’évo lution des PFNL. Elle est un document précieux consacré au développement des peuples par la promotion des PFNL en Afrique centrale en vue du renforcement de la sécurité alimentaire et la lutte contre la pauvreté. \n\n Voir aussi la sommaire en version anglais", + "abstractNote": "Ce livre nous emmène au cœur des zones de forêts denses et sahéliennes de l’Afrique centrale, un écosystème précieux et essentiel à la vie quotidienne de ses habitants, représentant l’un des trois principaux ensembles boisés tropicaux de la planète. Dix pays (Burundi, Cameroun, Congo, Gabon, Guinée Equatoriale, République Centrafricaine, République Démocratique du Congo, Rwanda, Sao Tomé & Principe, Tchad) abritent ces forêts et savanes, riches d’importantes ressources naturelles. Ils ont en com mun une longue histoire liée à la colonisation, suivie d'une expérience de coopération multiforme depuis les indépendances qui évolue incontestablement vers une intégration économique et monétaire. De nos jours, alors que les équilibres séculaires entre l’homme et la nature semblent ébranlés, que la sécurité alimentaire, la lutte contre la pauvreté et la préservation de la biodiversité et des ressources forestières sont devenus des enjeux mondiaux ; à l’heure où la croissance démographique non m aîtrisée fragilise le maintien des écosystèmes forestiers tout en accentuant les conflits liés à la recherche d’espace vital, le phénomène des changements climatiques vient davantage sonder le génie créateur des populations forestières dans la préservation et la gestion durable de la forêt et des produits forestiers non ligneux (PFNL) qui en sont issus. Cette publication est l’œuvre du personnel technique de la FAO, avec la contribution des partenaires internationaux et locaux engagés dans l’évo lution des PFNL. Elle est un document précieux consacré au développement des peuples par la promotion des PFNL en Afrique centrale en vue du renforcement de la sécurité alimentaire et la lutte contre la pauvreté.\n\nVoir aussi la sommaire en version anglais", "language": "fr", "libraryCatalog": "FAO Publications", "numPages": "251", "place": "Rome, Italy", "publisher": "FAO", "series": "Produits Forestiers Non-Ligneux", - "seriesNumber": "21", - "url": "http://www.fao.org/publications/card/fr/c/77dbd058-8dd4-4295-af77-23f6b28cc683/", + "seriesNumber": "No. 21", + "url": "https://www.fao.org/documents/card/fr/c/77dbd058-8dd4-4295-af77-23f6b28cc683/", "attachments": [ { "title": "Full Text PDF", @@ -662,7 +929,7 @@ var testCases = [ }, { "type": "web", - "url": "http://www.fao.org/publications/card/zh/c/mw246ZH/", + "url": "https://www.fao.org/publications/card/zh/c/mw246ZH/", "defer": true, "items": [ { @@ -682,7 +949,7 @@ var testCases = [ "libraryCatalog": "FAO Publications", "place": "Rome, Italy", "publisher": "FAO", - "url": "http://www.fao.org/publications/card/zh/c/mw246ZH/", + "url": "https://www.fao.org/publications/card/zh/c/mw246ZH/", "attachments": [ { "title": "Full Text PDF", @@ -749,12 +1016,12 @@ var testCases = [ }, { "type": "web", - "url": "http://www.fao.org/publications/card/en/c/5014f143-be17-4b58-b90e-f1c6bef344a0/", + "url": "https://www.fao.org/documents/card/ar/c/c6c2c8d7-3683-53a7-ab58-ce480c65f36c/", "defer": true, "items": [ { "itemType": "book", - "title": "Climate-Smart Agriculture: A Call for Action: Synthesis of the Asia-Pacific Regional Workshop Bangkok, Thailand, 18 to 20 June 2015", + "title": "الخطوط التوجيهية الطوعية بشأن الحوكمة المسؤولة لحيازة الأراضي ومصايد الأسماك والغابات في سياق الأمن الغذائي الوطني", "creators": [ { "lastName": "FAO", @@ -762,17 +1029,15 @@ var testCases = [ "fieldMode": 1 } ], - "date": "2015", - "ISBN": "9789251088630", - "abstractNote": "This publication is a summary of the workshop held in Bangkok, Thailand from 18 to 20 June 2015 to promote the mainstreaming and up-scaling of Climate-Smart Agriculture in the region. Included in the report are successful case studies that agriculturists have been practicing as a means to address food security under adverse circumstances.", - "language": "en", + "date": "2012", + "ISBN": "9789256072771", + "abstractNote": "هذه الخطوط التوجيهية هي أول صكّ عالمي شامل خاص بالحيازات وإدارتها يُعدّ من خلال مفاوضات حكومية دولية. وتضع هذه الخطوط التوجيهية مبادئ ومعايير مقبولة دولياً للممارسات المسؤولة لاستخدام الأراضي ومصايد الأسماك والغابات وللتحكّم بها. وهي تعطي توجيهات لتحسين الأطر القانونية والتنظيمية والمتصلة بالسياسات التي تنظّم حقوق الحيازة ولزيادة شفافية نظم الحيازة وإدارتها ولتعزيز القدرات والإجراءات التي تتخذها الأجهزة العامة ومؤسسات القطاع الخاص ومنظمات المجتمع المدني وجميع المعنيين بالحيازات وإد ارتها. وتُدرج هذه الخطوط التوجيهية إدارة الحيازات ضمن السياق الوطني للأمن الغذائي وهي تسعى إلى المساهمة في الإعمال المطرد للحق في غذاء كافٍ والقضاء على الفقر وحماية البيئة وتحقيق التنمية الاجتماعية والاقتصادية المستدامة.", + "language": "ar", "libraryCatalog": "FAO Publications", - "numPages": "106", + "numPages": "40", "place": "Rome, Italy", - "publisher": "FAO Regional Office for Asia and the Pacific", - "series": "RAP Publication", - "shortTitle": "Climate-Smart Agriculture", - "url": "http://www.fao.org/publications/card/en/c/5014f143-be17-4b58-b90e-f1c6bef344a0/", + "publisher": "FAO", + "url": "https://www.fao.org/documents/card/ar/c/c6c2c8d7-3683-53a7-ab58-ce480c65f36c/", "attachments": [ { "title": "Full Text PDF", @@ -781,28 +1046,25 @@ var testCases = [ ], "tags": [ { - "tag": "climate-smart agriculture" - }, - { - "tag": "forestry" + "tag": "guidelines" }, { - "tag": "market gardens" + "tag": "أمن غذائي" }, { - "tag": "meetings" + "tag": "إقتصاديات الغابة" }, { - "tag": "sustainable agriculture" + "tag": "اقتصاد الصيد" }, { - "tag": "sustainable development" + "tag": "الحكم" }, { - "tag": "urban farmers" + "tag": "النوع الاجتماعي" }, { - "tag": "water harvesting" + "tag": "حيازة الأراضي" } ], "notes": [], @@ -812,12 +1074,12 @@ var testCases = [ }, { "type": "web", - "url": "http://www.fao.org/publications/card/ar/c/c6c2c8d7-3683-53a7-ab58-ce480c65f36c/", + "url": "https://www.fao.org/documents/card/en/c/5014f143-be17-4b58-b90e-f1c6bef344a0/", "defer": true, "items": [ { "itemType": "book", - "title": "الخطوط التوجيهية الطوعية بشأن الحوكمة المسؤولة لحيازة الأراضي ومصايد الأسماك والغابات في سياق الأمن الغذائي الوطني", + "title": "Climate-Smart Agriculture: A Call for Action: Synthesis of the Asia-Pacific Regional Workshop Bangkok, Thailand, 18 to 20 June 2015", "creators": [ { "lastName": "FAO", @@ -825,14 +1087,17 @@ var testCases = [ "fieldMode": 1 } ], - "date": "2012", - "abstractNote": "هذه الخطوط التوجيهية هي أول صكّ عالمي شامل خاص بالحيازات وإدارتها يُعدّ من خلال مفاوضات حكومية دولية. وتضع هذه الخطوط التوجيهية مبادئ ومعايير مقبولة دولياً للممارسات المسؤولة لاستخدام الأراضي ومصايد الأسماك والغابات وللتحكّم بها. وهي تعطي توجيهات لتحسين الأطر القانونية والتنظيمية والمتصلة بالسياسات التي تنظّم حقوق الحيازة ولزيادة شفافية نظم الحيازة وإدارتها ولتعزيز القدرات والإجراءات التي تتخذها الأجهزة العامة ومؤسسات القطاع الخاص ومنظمات المجتمع المدني وجميع المعنيين بالحيازات وإد ارتها. وتُدرج هذه الخطوط التوجيهية إدارة الحيازات ضمن السياق الوطني للأمن الغذائي وهي تسعى إلى المساهمة في الإعمال المطرد للحق في غذاء كافٍ والقضاء على الفقر وحماية البيئة وتحقيق التنمية الاجتماعية والاقتصادية المستدامة.", - "language": "ar", + "date": "2015", + "ISBN": "9789251088630", + "abstractNote": "This publication is a summary of the workshop held in Bangkok, Thailand from 18 to 20 June 2015 to promote the mainstreaming and up-scaling of Climate-Smart Agriculture in the region. Included in the report are successful case studies that agriculturists have been practicing as a means to address food security under adverse circumstances.", + "language": "en", "libraryCatalog": "FAO Publications", - "numPages": "40", + "numPages": "106", "place": "Rome, Italy", - "publisher": "FAO", - "url": "http://www.fao.org/publications/card/ar/c/c6c2c8d7-3683-53a7-ab58-ce480c65f36c/", + "publisher": "FAO Regional Office for Asia and the Pacific", + "series": "RAP Publication", + "shortTitle": "Climate-Smart Agriculture", + "url": "https://www.fao.org/documents/card/en/c/5014f143-be17-4b58-b90e-f1c6bef344a0/", "attachments": [ { "title": "Full Text PDF", @@ -841,25 +1106,28 @@ var testCases = [ ], "tags": [ { - "tag": "null" + "tag": "climate-smart agriculture" + }, + { + "tag": "forestry" }, { - "tag": "null" + "tag": "market gardens" }, { - "tag": "null" + "tag": "meetings" }, { - "tag": "null" + "tag": "sustainable agriculture" }, { - "tag": "أمن غذائي" + "tag": "sustainable development" }, { - "tag": "الحكم" + "tag": "urban farmers" }, { - "tag": "حيازة الأراضي" + "tag": "water harvesting" } ], "notes": [], From b3dc7f92b82ce99fc8e89d0a738fd6ebdbdf4448 Mon Sep 17 00:00:00 2001 From: zoe-translates <116055375+zoe-translates@users.noreply.github.com> Date: Mon, 3 Apr 2023 19:43:37 +0000 Subject: [PATCH 007/125] CI: Update extension button selector (#3003) --- .ci/pull-request-check/selenium-test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/pull-request-check/selenium-test.js b/.ci/pull-request-check/selenium-test.js index 2076a96ec2a..37da3a04c97 100755 --- a/.ci/pull-request-check/selenium-test.js +++ b/.ci/pull-request-check/selenium-test.js @@ -113,10 +113,10 @@ var allPassed = false; // No API to retrieve extension ID. Hacks, sigh. await driver.get("chrome://system/"); - await driver.wait(until.elementLocated({id: 'extensions-value-btn'}), 60*1000); + await driver.wait(until.elementLocated({id: 'btn-extensions-value'}), 60*1000); // Chrome 89+ has the extension list expanded by default try { - let extBtn = await driver.findElement({css: '#extensions-value-btn'}); + let extBtn = await driver.findElement({css: '#btn-extensions-value'}); await extBtn.click(); } catch (e) {} let contentElem = await driver.findElement({css: '#content'}); From e6c28bf9164199217d63b0eeef8b750aa83e87bc Mon Sep 17 00:00:00 2001 From: Abe Jellinek Date: Wed, 5 Apr 2023 11:26:39 -0400 Subject: [PATCH 008/125] Newspapers.com: Support /article/ pages, improve metadata https://forums.zotero.org/discussion/comment/432139/#Comment_432139 --- newspapers.com.js | 178 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 139 insertions(+), 39 deletions(-) diff --git a/newspapers.com.js b/newspapers.com.js index dd7caf6a490..abfa0f20ae1 100644 --- a/newspapers.com.js +++ b/newspapers.com.js @@ -2,14 +2,14 @@ "translatorID": "22dd8e35-02da-4968-b306-6efe0779a48d", "label": "newspapers.com", "creator": "Peter Binkley", - "target": "^https?://www\\.newspapers\\.com/clip/", + "target": "^https?://[^/]+\\.newspapers\\.com/(clip|article)/", "minVersion": "3.0", "maxVersion": "", "priority": 100, "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2020-10-29 03:32:09" + "lastUpdated": "2023-04-05 15:26:20" } /* @@ -39,23 +39,17 @@ function detectWeb(_doc, _url) { return "newspaperArticle"; } +function doWeb(doc, url) { + if (url.includes('/clip/')) { + scrapeClip(doc, url); + } + else { + scrapeArticle(doc, url); + } +} -function doWeb(doc, _url) { +function scrapeClip(doc, url) { var newItem = new Zotero.Item("newspaperArticle"); - var scripts = doc.getElementsByTagName("script"); - var json = ''; - var jsonre = /var staPageDetail = JSON.parse\((.+?)\);/; - for (var i = 0; i < scripts.length; i++) { - var arr = scripts[i].textContent.match(jsonre); - if (arr) { - json = arr[1]; - break; - } - } - - // one JSON.parse to unstringify the json string, and one to parse it into an object - // the replace fixes escaped apostrophes in the source, which JSON.parse considers invalid - var details = JSON.parse(JSON.parse(json.replace(/\\'/g, "'"))); var metaArr = {}; var metaTags = doc.getElementsByTagName("meta"); @@ -64,9 +58,9 @@ function doWeb(doc, _url) { metaArr[metaTag.getAttribute("property")] = metaTag.getAttribute("content"); } } - newItem.title = details.citation.title; + newItem.title = text(doc, '#mainContent h1') || text(doc, '[itemprop="about"]'); // remove the unnecessary xid param - newItem.url = details.citation.url.replace(/\?xid=[0-9]*$/, ""); + newItem.url = attr(doc, 'link[rel="canonical"]', 'href'); /* The user can append the author to the title with a forward slash @@ -82,41 +76,69 @@ function doWeb(doc, _url) { newItem.creators.push(Zotero.Utilities.cleanAuthor(author, "author")); } } - - newItem.abstractNote = details.media.note; - var uniqueID = newItem.url.match(/\/clip\/(\d+)/)[1]; - var pdfurl = "https://www.newspapers.com/clippings/download/?id=" + uniqueID; - newItem.attachments.push({ - title: "Full Text PDF", - mimeType: "application/pdf", - url: pdfurl - }); - - newItem.publicationTitle = details.source.publisherName; + newItem.publicationTitle = text(doc, '[itemprop="name"]'); // details["source"]["title"] gives a string like // "Newspapers.com - The Akron Beacon Journal - 1939-10-30 - Page Page 15" - var editiontokens = details.source.title.replace(/ - /g, "|").split("|"); - if (editiontokens.length == 3) { // there's an edition label - newItem.edition = editiontokens[1]; - } - newItem.pages = editiontokens.slice(-1)[0].replace(/Page/g, ''); - newItem.date = details.source.publishedDate; - newItem.place = details.source.publishedLocation; + newItem.pages = text(doc, '[itemprop="position"]').replace(/Page/g, ''); + newItem.date = ZU.strToISO(text(doc, '[itemprop="dateCreated"]')); + newItem.place = text(doc, '[itemprop="locationCreated"]'); + + newItem.attachments.push(makeImageAttachment(url)); + newItem.attachments.push(makePDFAttachment(url)); // handle empty title if (newItem.title === "") { - newItem.title = "Clipped From " + newItem.publicationTitle; + newItem.title = "Article clipped from " + newItem.publicationTitle + ""; } newItem.complete(); } +function scrapeArticle(doc, url) { + let item = new Zotero.Item('newspaperArticle'); + let json = JSON.parse(text(doc, 'script[type="application/ld+json"]')); + + item.publicationTitle = json.publisher && ZU.unescapeHTML(json.publisher.legalName); + item.title = ZU.trimInternal(ZU.unescapeHTML(json.about)) + || 'Article clipped from ' + item.publicationTitle + ''; + item.abstractNote = ZU.unescapeHTML(json.text); + item.place = ZU.unescapeHTML(json.locationCreated); + item.date = json.datePublished; + item.pages = json.pageStart && ZU.unescapeHTML(json.pageStart.replace('Page', '')); + item.url = attr(doc, 'link[rel="canonical"]', 'href'); + item.attachments.push(makeImageAttachment(url)); + item.attachments.push(makePDFAttachment(url)); + + item.complete(); +} + +function getID(url) { + return url.match(/\/(\d+)/)[1]; +} + +function makePDFAttachment(url) { + return { + title: 'Full Text PDF', + mimeType: 'application/pdf', + url: 'https://www.newspapers.com/clippings/download/?id=' + getID(url) + }; +} + +function makeImageAttachment(url) { + return { + title: 'Image', + mimeType: 'image/jpeg', + url: 'https://img.newspapers.com/img/img?clippingId=' + getID(url) + }; +} + /** BEGIN TEST CASES **/ var testCases = [ { "type": "web", "url": "https://www.newspapers.com/clip/7960447/my-day-eleanor-roosevelt/", + "detectedItemType": "newspaperArticle", "items": [ { "itemType": "newspaperArticle", @@ -135,6 +157,10 @@ var testCases = [ "publicationTitle": "The Akron Beacon Journal", "url": "https://www.newspapers.com/clip/7960447/my-day-eleanor-roosevelt/", "attachments": [ + { + "title": "Image", + "mimeType": "image/jpeg" + }, { "title": "Full Text PDF", "mimeType": "application/pdf" @@ -149,10 +175,11 @@ var testCases = [ { "type": "web", "url": "https://www.newspapers.com/clip/18535448/the-sunday-leader/", + "detectedItemType": "newspaperArticle", "items": [ { "itemType": "newspaperArticle", - "title": "Clipped From The Sunday Leader", + "title": "Article clipped from The Sunday Leader", "creators": [], "date": "1887-07-17", "libraryCatalog": "newspapers.com", @@ -161,6 +188,10 @@ var testCases = [ "publicationTitle": "The Sunday Leader", "url": "https://www.newspapers.com/clip/18535448/the-sunday-leader/", "attachments": [ + { + "title": "Image", + "mimeType": "image/jpeg" + }, { "title": "Full Text PDF", "mimeType": "application/pdf" @@ -175,6 +206,7 @@ var testCases = [ { "type": "web", "url": "https://www.newspapers.com/clip/31333699/driven-from-governors-office-ohio/", + "detectedItemType": "newspaperArticle", "items": [ { "itemType": "newspaperArticle", @@ -187,6 +219,74 @@ var testCases = [ "publicationTitle": "Rushville Republican", "url": "https://www.newspapers.com/clip/31333699/driven-from-governors-office-ohio/", "attachments": [ + { + "title": "Image", + "mimeType": "image/jpeg" + }, + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.newspapers.com/article/the-times-picayune-telegraphed-to-the-ne/120087578/", + "detectedItemType": "newspaperArticle", + "items": [ + { + "itemType": "newspaperArticle", + "title": "Telegraphed to the New Orleans Picayune. Latest from Charleston. Fort Sumter Returns Fire", + "creators": [], + "date": "1861-04-13", + "abstractNote": "Telegraphed to the New Orleans Picayune. LATEST FROM CHARLESTON. FORT SUMTER RETflUS FIRE. SULLI VAN12AND MORRIS ISLAND BATTERIES AT WORK. BREACH MADE IN FORT SUMTER. War Vessels Reported Outside. By the Southwestern Line. Charleston, April 12. The batteries of Sullivan's Island, Morris Island and other points opened fire on Fort Sumter at half - past four o'clock this morning. Fort Sumter returned the fire. A brisk cannonading is being kept up. There is no infoimation from the seaboard. The military are under arms. The whole population is on the streets, and the harbor is filled with anxious spectators. SECONB DISPATCH. The Moating battery is doing good service. Up to eleven o clock there has been no loea on our side. Fort Sumter replied at 7 o'clock this morning, and has kept up an astonishing fire ever since. Stevens's battery is slightly injured. Three sbejls are fired per minute. Four hundred, in all, have fallen. A breach is expected to be made in Fort Sumter to - morrow. Major Anderson's fire is principally directed I against the floating battery. j War vessels are reported outside the harbor. Only two soldiers are wounded on Salli - ! van's Island. The range is more perfect from the land batteries. Every shot tells. It ia thought from Mnjor Anderson's fire thai he haa more men than was supposed. Fort Sumter will succumb by to - morrow. It is raining at Charleston, but there - is no cessation of the batteries. A continuous steady fire on both sides is beinc kept up. The cutter Harriet Lane, and the steam gnu boat Crntader, are reported olf the bar, but have not entered the harbor. The War Department have as yet no official diepatches. (Jen. Beauregard was at the batteries all day. , The Government expects Fort Sumter to succumb to - morrow. third dispatch The firing continued all day. Two of Fort Sumter's guns are silenced, and it is reported a breach has been made through the southeast wall. No casualty has yet happened to any of the forces. Only seven of the nineteen batteries have opened fire on Fort Sumter. The remainder are held ready for the expected fleet. Two thousand men reached the city this morning and immediately embarked for Morris Island. FOURTH DI fAT H. Charleston, April 10, 11 P. M. Tne bombardment of Fort Saniter is going on every twenty minutes from the mortars It is supposed Major Anderson is resting his men for the night. Three vessels of war are reported outside tho bar. They cannot get in on account of the roughness of the sea. No one has as yet received any injury. The floating battery works admirably well. Every inlet to the harbor is well guarded. Our forces are having a lively time of it.", + "libraryCatalog": "newspapers.com", + "pages": "4", + "place": "New Orleans, Louisiana", + "publicationTitle": "The Times-Picayune", + "url": "https://www.newspapers.com/article/the-times-picayune-telegraphed-to-the-ne/120087578/", + "attachments": [ + { + "title": "Image", + "mimeType": "image/jpeg" + }, + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://nydailynews.newspapers.com/article/daily-news/121098969/", + "detectedItemType": "newspaperArticle", + "items": [ + { + "itemType": "newspaperArticle", + "title": "Article clipped from Daily News", + "creators": [], + "date": "1965-02-26", + "abstractNote": "Donavena 8-5 Choice; Can He Kayo Folley? By Jim McCulIey Professional oddsmakers, otherwise referred to as bookies, evidently are counting on Oscar (Ringo) Bona- vena to flatten Zora Folley the Garden. Otherwise, why young thumper from Argentina an 8-5 favorite over the ringwise No. 5 heavyweight contender from Arizona? Only two fighters in Folley's 13-year career have outpointed him England's Henry Cooper, in London, and big Ernie Terrell, in New York. Five men have stopped Zora, however. IT DOESN'T SEEM possible Bonavena, with only eight pro fights under his belt, could win a decision over tonight's 32-year-old opponent. Oscar has stopped seven of his eight opponents, however, and, of course, does have a powerful body and a punishing punch in either mitt. The fight mob is really puzzled over this fight. Some of those well versed in fisticuffs can't understand how the odds-bodkins can make 22-year-old Bonavena such a big favorite. Some 10,000 fans are expected to come see for themselves and put another $40,- 000 into the current boxing revival. \"I KNOW FOLLEY dogs it at times,\" said a former heavyweight contender, who did not want to be named because he is now an official with the boxing commission. ''But Bonavena is a real novice compared to Zora. It seems to me Folley should be a big favorite, but then the kid does have a punch and he is game. It's possible he can reach Folley and knock him out.\" The price, for Folley backers, Is most enticing. \"I CAN'T RESIST the price,\" said a knowledgeable fight man who has been known to wager a bob now and then when the figures are right. \"Know something, 1 think it will be down close to pick 'em before they get into the ring.\" One thing is certain. Folley can't lose another fight in New York at this time, or he is through as a top contender. He is going for a payday on the gamble that he can go the distance with Oscar; and there is a chance he might stop the young man, too, though nobody has done that yet. RIGHT NOW, FOLLEY is unbeaten in hi3 last six bouts since losing to Terrell here July 27, '63. In that span he has whipped George Chuvalo, easily, and has recorded a draw with European champion Karl Maldenbfrger in Germany. Zora's overall record stands 68-7-4, for 79 professional fights, and includes 38 knockouts, some proof that he can punch as well as box. Only opponent to go the route (10 rounds) with Bonavena was Dick Wipperman, last Nov. 13 here. Oscar came back to the Garden a month later and knocked out Billy Stephan in six. The South American still is unranked among the big boys, but a win tonight will put him up there where he can start hollering. History shows heavyweights do mature a lot quicker than the lighter men, and Oscar may ev.en. be an unusual young fighter, v . . ( . r in 10 rounds or less tonight at would they continue to list the - Vlsic lliv ;.. Vnn-t ST!rt -. lM. FEB. 26, 1958 ZDhe BOSTON CELTICS WOM THEIR SECOMt STRAIGHT N.&.A. EASTERN CROWN BY DOWNING DETROIT, 106-99, AS &1LL RUSSELL COUTftOLU&THE BOARDS. BOBCOUSYAN& BILL SHAfeMAH EACH SCORED 18 POIWTS. Lincoln Downs Results 1ST Clmp.: 4-np: 5 f.: off 1:33. Ravenala Prince (Garry)5.ti0 4i 2 SO Mission Bound (Parker) 6.10 .'i.8'1 Favorite Act (Bradley) K.MI T-l:02, Also Lord Culpeper. Your Reporter, Deacon Shnne. Prmrie Rose. Rinsr Shut, Fearless Leader, ilaryg Gilt. Soft Glance. 2D Clmg-.; 4-np: 7 f.: off 2:00. Idle Threats (Allan) 4 no 2 SO Grey Whirl (Giovanni) 3.40 3.00 Good Effort (Maeda) B.20 T-1:32t4. Also Greek Paire. Inquisition. Frozen North, Fast Bid. Foxy Sway. (Daily Double. 8-1, Paid :!.\". liOl 3D Clm?:3yrs:mdns:5 f :off 2 :2!) . Dogrwood Pateh(MaRia)7.ai) o.no 4.20 I.L Abie K. t Bradley) 13. NO U.KO Peaceful T. (Donahue) H.uO T.-l:t)3. Also Doe I.ark. AlHnx. Miss Pilot. Sum Bomb. Fast Bell. Greek Action, Win Joe. Dont Btatne Babe. 4TH Clmar.: 4-up; 7 t.: off 2:58. Irish Dotty (Bradley) 4.4D 3.20 2. SO Sibling- (Allan) 9.80 6.20 Brimstone Road (Row an 6. Of) T.-l :35 . Also Stahlstown. Emerson Hill. Patti Dowd. Ou The Lawn. Sieve H.. Game Start. Set. 5TH Clma:.: 3-up: 8 t.; off 3:254. Ancient Queen (Lamonte)-4.80 3. no 2.40 Wlwndilly (Merrier) 3 20 2 .So Lady Mink (Bradley) 2.80 T-l:02. Alio Mandolas. Lady Rhody. O. K. Debbie. Jury Verdict. Swift Salonga. Mix n Match. La Calvados. 6TH Clm?: 3-4 yrs; 5 f: off 3:52. Tessie Tansor(Davern)12.60 o.BO 5.00 French Line (Myers) 4.80 5 40 Captain Bronze (Allan) 10. hi) T.-l:02 9i. Alyso Rosie Anirel. Lony-bridge Lu Lu. Star Status, Toute Ma Vie. Tompkins County. 7TH Alw.: 3-4-yos.: 5 fur. off 4:20. Lories Honey (Hole) 24.20 20 3.8\" Rndoon (Clinch) 2.40 2.40 Presta Sun (Gamb'della) 5.00 T.-l:03. Also Green Toea. Anthony Scarfo. Prince O Morn. Captain Lockitup. Caronia. 8TH Clmr.: 4-up: 1 m.: off 4:48. ratcount (Alberts) 13.HO 5 HO 4.20 Lone Peak (Rodriguez) 5.60 3 flu Kilda (Ledezma) 3.40 T-l:48Si. Also Hue or Spank. Carb-anrel, Whitey. Wild Desire. 9TH Clmg-: 41iip: 1 m: off 5:16. Oportscaster (Allan) 20.80 8.KO 7.20 Waste Of Time(Miller) 49.20 2B.20 Da.vFromDallas(G's'do) 20.40 T.-1:5H4. Also Symboleer, Dandy Randy. Sea Tread. My Buyer. Cosmic Rule. Busted Budeet. Another Take, Presented. (Twin Double 8-1 8-3 Paid $3.51 1.20) , Att, 4,744. Handle $364,968. ' r think ( ConraoLf) THEY'LL 7t1-fT EVER SvSXv. ' C COME J uAV' BE A LOHG JfeTV", + "libraryCatalog": "newspapers.com", + "pages": "60", + "place": "New York, New York", + "publicationTitle": "Daily News", + "url": "https://www.newspapers.com/article/daily-news/121098969/", + "attachments": [ + { + "title": "Image", + "mimeType": "image/jpeg" + }, { "title": "Full Text PDF", "mimeType": "application/pdf" From 7f8c06b5f7455b6fa075c113114605cb167d76c6 Mon Sep 17 00:00:00 2001 From: Sebastian Karcher Date: Fri, 7 Apr 2023 04:39:25 -0400 Subject: [PATCH 009/125] INSPIRE: Fix PDF import (#3011) add defer to item tests. https://forums.zotero.org/discussion/comment/432186#Comment_432186 --- INSPIRE.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/INSPIRE.js b/INSPIRE.js index b7ac6984bc4..a281abef57f 100644 --- a/INSPIRE.js +++ b/INSPIRE.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2021-07-19 16:22:40" + "lastUpdated": "2023-04-06 18:53:02" } /* @@ -88,7 +88,7 @@ function scrape(doc, url) { item.tags.push({ tag: tag.textContent.trim() }); } - for (let action of doc.querySelectorAll('.__ListItemAction__ a')) { + for (let action of doc.querySelectorAll('.__UserAction__ a')) { if (/\bpdf\b/i.test(action.textContent)) { item.attachments.push({ title: 'Full Text PDF', @@ -115,6 +115,7 @@ var testCases = [ { "type": "web", "url": "https://inspirehep.net/literature/1284987", + "defer": true, "items": [ { "itemType": "journalArticle", @@ -186,6 +187,7 @@ var testCases = [ { "type": "web", "url": "https://inspirehep.net/literature/1282171", + "defer": true, "items": [ { "itemType": "journalArticle", From 48a7462fea880d1032222b30aca4698491014f30 Mon Sep 17 00:00:00 2001 From: Emiliano Heyns Date: Sun, 9 Apr 2023 20:55:17 +0200 Subject: [PATCH 010/125] BibTex fix mark-up conversion * fixes #3013 * use reverseMappingTable if possible * Update tests & add new one --- BibTeX.js | 55 +++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/BibTeX.js b/BibTeX.js index 8d15b6a7275..7aef0de5d41 100644 --- a/BibTeX.js +++ b/BibTeX.js @@ -18,7 +18,7 @@ }, "inRepository": true, "translatorType": 3, - "lastUpdated": "2022-10-31 23:11:08" + "lastUpdated": "2023-04-09 18:35:07" } /* @@ -684,6 +684,7 @@ function unescapeBibTeX(value) { value = value.replace(mapped, unicode); } } + value = value.replace(/\$([^$]+)\$/g, '$1') // kill braces value = value.replace(/([^\\])[{}]+/g, "$1"); @@ -1109,15 +1110,23 @@ function mapHTMLmarkup(characters){ return characters; } - +function xcase(prefix, cased, tag, tex) { + return (prefix ? `$${prefix}$` : '') + (reversemappingTable[`$${tex}{${cased}}$`] || `<${tag}>${cased}`) +} +function sup(match, prefix, cased) { + return xcase(prefix, cased, 'sup', '^'); +} +function sub(match, prefix, cased) { + return xcase(prefix, cased, 'sub', '_'); +} function mapTeXmarkup(tex){ //reverse of the above - converts tex mark-up into html mark-up permitted by Zotero //italics and bold tex = tex.replace(/\\textit\{([^\}]+\})/g, "$1").replace(/\\textbf\{([^\}]+\})/g, "$1"); //two versions of subscript the .* after $ is necessary because people m - tex = tex.replace(/\$[^\{\$]*_\{([^\}]+\})\$/g, "$1").replace(/\$[^\{]*_\{\\textrm\{([^\}]+\}\})/g, "$1"); + tex = tex.replace(/\$([^\{\$]*)_\{([^\}]+)\}\$/g, sub).replace(/\$([^\{\$]*)_\{\\textrm\{([^\}\$]+)\}\}\$/g, sub); //two version of superscript - tex = tex.replace(/\$[^\{]*\^\{([^\}]+\}\$)/g, "$1").replace(/\$[^\{]*\^\{\\textrm\{([^\}]+\}\})/g, "$1"); + tex = tex.replace(/\$([^\{\$]*)\^\{([^\}]+)\}\$/g, sup).replace(/\$([^\{\$]*)\^\{\\textrm\{([^\}]+)\}\}\$/g, sup); //small caps tex = tex.replace(/\\textsc\{([^\}]+)/g, "$1"); return tex; @@ -3326,7 +3335,7 @@ var testCases = [ "items": [ { "itemType": "journalArticle", - "title": "Test of markupconversion: Italics, bold, superscript, subscript, and small caps: Mitochondrial DNA2$ sequences suggest unexpected phylogenetic position of Corso-Sardinian grass snakes (Natrix cetti) and do not support their species status, with notes on phylogeography and subspecies delineation of grass snakes.", + "title": "Test of markupconversion: Italics, bold, superscript, subscript, and small caps: Mitochondrial DNA₂ sequences suggest unexpected phylogenetic position of Corso-Sardinian grass snakes (Natrix cetti) and do not support their species status, with notes on phylogeography and subspecies delineation of grass snakes.", "creators": [ { "firstName": "U.", @@ -3348,7 +3357,7 @@ var testCases = [ "DOI": "10.1007/s13127-011-0069-8", "itemID": "Frit2", "pages": "71-80", - "publicationTitle": "Actes du ème$ Congrès Français d'Acoustique", + "publicationTitle": "Actes du 4ème Congrès Français d'Acoustique", "volume": "12", "attachments": [], "tags": [], @@ -4159,6 +4168,40 @@ var testCases = [ "seeAlso": [] } ] + }, + { + "type": "import", + "input": "@article{Borissov:2855446,\r\n author = \"Borissov, Alexander and Solokhin, Sergei\",\r\n collaboration = \"ALICE\",\r\n title = \"{Production of $\\Sigma^{0}$ Hyperon and Search of\r\n $\\Sigma^{0}$ Hypernuclei at LHC with ALICE}\",\r\n journal = \"Phys. At. Nucl.\",\r\n volume = \"85\",\r\n number = \"6\",\r\n pages = \"970-975\",\r\n year = \"2023\",\r\n url = \"https://cds.cern.ch/record/2855446\",\r\n doi = \"10.1134/S1063778823010131\",\r\n }", + "items": [ + { + "itemType": "journalArticle", + "title": "Production of Σ⁰ Hyperon and Search of Σ⁰ Hypernuclei at LHC with ALICE", + "creators": [ + { + "firstName": "Alexander", + "lastName": "Borissov", + "creatorType": "author" + }, + { + "firstName": "Sergei", + "lastName": "Solokhin", + "creatorType": "author" + } + ], + "date": "2023", + "DOI": "10.1134/S1063778823010131", + "issue": "6", + "itemID": "Borissov:2855446", + "pages": "970-975", + "publicationTitle": "Phys. At. Nucl.", + "url": "https://cds.cern.ch/record/2855446", + "volume": "85", + "attachments": [], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] } ] /** END TEST CASES **/ From 5bf94391d43da8a2a1119743b6ab69e40a1b90d3 Mon Sep 17 00:00:00 2001 From: zoe-translates <116055375+zoe-translates@users.noreply.github.com> Date: Tue, 11 Apr 2023 21:51:35 +0000 Subject: [PATCH 011/125] Add translator for Lapham's Quarterly (#2996) --- Lapham's Quarterly.js | 1062 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1062 insertions(+) create mode 100644 Lapham's Quarterly.js diff --git a/Lapham's Quarterly.js b/Lapham's Quarterly.js new file mode 100644 index 00000000000..884d1cdd5c3 --- /dev/null +++ b/Lapham's Quarterly.js @@ -0,0 +1,1062 @@ +{ + "translatorID": "e329ec79-397e-4aa5-a06e-1aa32f10a138", + "label": "Lapham's Quarterly", + "creator": "Zoë C. Ma", + "target": "^https?://www\\.laphamsquarterly\\.org/", + "minVersion": "5.0", + "maxVersion": "", + "priority": 100, + "inRepository": true, + "translatorType": 4, + "browserSupport": "gcsibv", + "lastUpdated": "2023-04-11 10:35:51" +} + +/* + ***** BEGIN LICENSE BLOCK ***** + Copyright © 2023 Zoë C. Ma + + This file is part of Zotero. + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + ***** END LICENSE BLOCK ***** +*/ + +function detectWeb(doc, url) { + const urlObj = new URL(url); + // About pages, legal notes, content listings, event notices... or + // content pages without identifiable author, or fragmentary quotations of + // historical materials. + const skipPath = /^\/(about|legal|issues|archive|contributors|conversations|lq-interactives|outreach|programs|events|world-in-time|deja-vu)/; + if (urlObj.pathname.match(skipPath)) { + return false; + } + + // Also skip pages from magazine sections that has no usable author + // info. (Maps, Miscellany, charts and graphs, etc.). + const skipSection = /^\/.+\/(maps|miscellany|charts-graphs)\/.+/; + if (urlObj.pathname.match(skipSection)) { + return false; + } + + // Also skip the individual issue pages. This can only be done by + // inspecting the document. + if (doc.querySelector("body.node-type-issue")) { + return false; + } + + if (urlObj.pathname.match(/^\/search\/node\/.+/)) { + // Search results. + if (getSearchResults(doc, true)) { + return "multiple"; + } + else { + return false; + } + } + + if (doc.querySelector("body.node-type-podcast")) { + return "podcast"; + } + + if (doc.querySelector("body.section-roundtable")) { + return "blogPost"; + } + + return "magazineArticle"; +} + +async function doWeb(doc, url) { + if (detectWeb(doc, url) == 'multiple') { + const items = await Z.selectItems(getSearchResults(doc, false)); + if (!items) return; + for (const url of Object.keys(items)) { + await scrape(await requestDocument(url)); + } + } + else { + await scrape(doc, url); + } +} + +function getSearchResults(doc, checkOnly = false) { + const resultElems = doc.querySelectorAll(".search-results .search-result"); // Lovely semantics! + if (!resultElems.length) return false; + + // Title string -> array of URLs. + const titleMap = new Map(); + // While collecting title -> URL mapping for possible duplicate titles, + // check for duplicate URLs too, though this is unlikely. If it does + // happen, the first one in the document order takes precedence. + const hrefsSeen = new Set(); + for (const elem of resultElems) { + const href = attr(elem, "h3 > a", "href"); + const title = text(elem, "h3 > a"); + + if (href && !hrefsSeen.has(href) && title) { + if (checkOnly) return true; + + hrefsSeen.add(href); + // Title may contain duplicates even if the links are + // unique. In other words, one title may be associated with + // multiple (i.e. an array of) URLs. + if (!titleMap.has(title)) { + titleMap.set(title, []); + } + titleMap.get(title).push(href); + } + } + + // If the same title text is associated with multiple URLs, add a + // parenthesized number showing the order of the title's appearance in the + // search results. + const items = {}; + // Map conveniently maintains insertion order. + for (const [title, hrefArray] of titleMap) { + const hasDup = hrefArray.length > 1; + for (const [i, href] of hrefArray.entries()) { + items[href] = !hasDup + ? title + : `${title} (${i + 1}, URL: ${(new URL(href)).pathname})`; + } + } + + return hrefsSeen.size && items; +} + +async function scrape(doc, url = doc.location.href) { + const type = detectWeb(doc, url); + if (!type) { + // This could happen if the user selects an item from the + // multiple, but that item happens to be something we cannot + // exclude based on URL/title alone. + Z.debug(`scrape function encountered mismatched type ${type} for ${url}`); + return; + } + + const item = new Z.Item(type); + item.url = url; + item.language = attr(doc, "html", "lang"); + item.attachments = []; + + switch (type) { + case "magazineArticle": + await applyMagazine(doc, item); + break; + case "blogPost": + applyBlog(doc, item); + break; + case "podcast": + applyPodcast(doc, item); + break; + } + + item.attachments.push({ + document: doc, + title: "Snapshot", + mimeType: "text/html" + }); + + item.complete(); +} + +// Magazine articles. This will always be async even if the async task is not +// performed, in the (unlikely) case when the issue-info URL to be scraped is +// not found on the article page. +async function applyMagazine(doc, item) { + item.ISSN = "1935-7494"; + item.publicationTitle = "Lapham’s Quarterly"; + + item.title = text(doc, "#page-title"); + item.creators = parseAuthors(getArticleAuthorText(doc)); + + const excerpt = text(doc, ".excerpt"); + if (excerpt) item.abstractNote = excerpt; + + if (doc.querySelector("body.node-type-voices-in-time")) { + // Voices in Time + let tmp = ZU.trimInternal(text(doc, ".title .date")); // Original date + if (tmp) { + item.originalDate = tmp; + } + + tmp = getVITRightsTrans(doc); // Rights and translators + if (tmp) { + if (tmp.rights) { + item.rights = tmp.rights; + } + if (tmp.translators) { // could be undefined + item.creators.push(...tmp.translators); + } + } + + tmp = getVITAboutText(doc); // "About the text" or brief bio of author + if (tmp) { + item.notes = [tmp]; + } + } + + const issueRelURL = attr(doc, ".sticky-content > a", "href"); + if (!issueRelURL) { + Z.debug(`Article at ${item.url} missing the link to its issue.`); + return undefined; + } + + const issueURL = (new URL(issueRelURL, doc.location)).href; + return setIssueDate(issueURL, item); +} + +// Cache for the issue info. Keys are the permalinks to the issue-page URLs, +// and values are the corresponding issue info returned by +// fetchIssueDateInfo(). This is to avoid repeated network requests for the +// same document when saving multiple items. +const _issueCache = new Map(); + +async function setIssueDate(url, item) { + let value; + if (_issueCache.has(url)) { + value = _issueCache.get(url); + } + else { + value = await fetchIssueDateInfo(url); + if (value) { + _issueCache.set(url, value); + } + } + Object.assign(item, value); +} + +async function fetchIssueDateInfo(url) { + let doc; + try { + doc = await requestDocument(url); + } + catch (err) { + Z.debug(`Failed to request ${url} for issue/date info.`); + return null; + } + + let dateText = text(doc, "p.date"); + if (!dateText) { + Z.debug(`Issue/date info unexpectedly missing at ${url}`); + return null; + } + + dateText = ZU.trimInternal(dateText); + + // dateText should look like the following: + // "Volume XIV, Number 4 | [season/month-range] 2022". + // Convert it into array like ["XIV", "4", "2022"] + const [volume, number, year] = dateText.split(/[,|]/) + .map(x => x.trim().split(" ")[1]); + return { + volume: romanToInt(volume), + issue: parseInt(number), + date: parseInt(year) + }; +} + +// Get the rights and translator info for Voices in Time if any. +function getVITRightsTrans(doc) { + const paragraphs = doc.querySelectorAll(".content-wrapper > p"); + if (!paragraphs.length) { + return false; + } + + const str + = ZU.trimInternal(paragraphs.item(paragraphs.length - 1).textContent); + + if (str) { + const infoObj = {}; + + // . [optional words ](C) yyyy[ by name] ... (full stop) + let match = str.match(/(?:^|\.\s+)((?:\w+\s+)*©\s+\d+.+?\.)/im); + if (match) { + infoObj.rights = match[1]; + } + + // Translator. "Translated by ... [stop or semicolon]" + match = str.match(/(?:^|\.\s+)translated by (.+?)[.;]/i); + if (match) { + const transArray = parseAuthors(match[1], "translator"); + if (transArray.length) { + infoObj.translators = transArray; + } + } + + return infoObj; + } + + return false; +} + +// Get the text block under "About this text" for Voices in Time. The block is +// present even if the text has no identifiable author. +function getVITAboutText(doc) { + const paragraphs = doc.querySelectorAll(".bio-block > p"); + if (!paragraphs.length) { + return ""; + } + + const output = []; + for (const paragraph of paragraphs.values()) { + output.push(ZU.trimInternal(paragraph.textContent.trim())); + } + // Re-inserting paragraph-ending line breaks and add extra line break + // between paragraphs. + return output.join("\n\n"); +} + +// Blog articles. +function applyBlog(doc, item) { + // Blog-article title proper + item.title = text(doc, ".title > h2"); + item.creators = parseAuthors(getArticleAuthorText(doc)); + item.date = getBlogPostDate(doc); + // blogTitle refers to the name of the blog hosted by Lapham's. + item.blogTitle = text(doc, "#page-title"); +} + +function getBlogPostDate(doc) { + const dateText = text(doc, ".pub-date"); + return !!dateText && (new Date(dateText)).toISOString(); +} + +// Returns the author string (for magazine article or blog post). +function getArticleAuthorText(doc) { + // Take the author's byline from the "Contributor" block, which is more + // cumbersome but also more reliable than the byline at ".title .author". + let byline = text(doc, + '.banner-block a[href^="/contributors/"]' // usual place + + ', .bio-heading a[href^="/contributors/"]' // "voices in time" + ); + + if (!byline) { + // Just in case the above didn't work, try this more obvious but less + // generic one. + byline = text(doc, ".title .author"); // Could be p or h2 element. + + // NOTE: failure mode: None of the selectors can locate the element. + if (!byline) return ""; + + // Remove any initial "By ..." + byline = byline.replace(/^(By\s+)?/i, ""); + } + + const authorText = ZU.trimInternal(byline); + + if (authorText === "Lapham’s Quarterly") { + // Skip adding author info when the "author" is the same as the + // publisher. + return ""; + } + + return authorText; +} + +// Podcasts +function applyPodcast(doc, item) { + const podPublication = text(doc, ".title > h1"); + item.seriesTitle = podPublication; + // Date text uses the same DOM element as it is on blog articles. + item.date = getBlogPostDate(doc); + let t = getPodDuration(doc); + if (!Number.isNaN(t)) { + item.runningTime = t; + } + + const mainAudioSelector = ".top-image-block audio > source"; + const epURL = attr(doc, mainAudioSelector, "src"); + item.audioFileType = attr(doc, mainAudioSelector, "type"); + + item.abstractNote = attr(doc, "meta[name='description']", "content"); + + const headingText = text(doc, ".title > h2"); + if (podPublication.toLowerCase() === "the world in time") { + // The EiC's own podcast. + item.creators = [ZU.cleanAuthor("Lewis H. Lapham", "author")]; + item.title = headingText; + // Extract episode number + const epMatch = epURL.match(/episode-(\d+)-/i); + if (epMatch) { + item.episodeNumber = parseInt(epMatch[1]); + } + + const guestName = inferEiCPodGuest(doc, headingText, epURL); + if (guestName) { + item.creators.push(ZU.cleanAuthor(guestName, "guest")); + } + } + else if (podPublication.toLowerCase() === "lq podcast") { + // The metadata for "LQ Podcast" is more difficult to obtain. The + // naming scheme is more diverse, and even if we're tempted to parse + // the audio filename for author info, see this for how it may not + // work: https://www.laphamsquarterly.org/content/poes-terror-soul + const [, ep, title] = headingText.match(/#(\d+)\s+(.+)/i); + item.episodeNumber = parseInt(ep); + item.title = title; + } + + item.attachments.push({ + title: "Audio", + mimeType: item.audioFileType, + url: epURL, + }); +} + +// Get the duration of episode as a string. This can return NaN if the duration +// cannot be scraped from the doc. +function getPodDuration(doc) { + const currTime = text(doc, ".jp-current-time"); + const remainTime = text(doc, ".jp-duration"); // Negative value. + return timeToDuration(parseTime(currTime) - parseTime(remainTime)); +} + +// Parse mm:ss time duration string as number of seconds. Returns NaN if the +// input string does not match the expected format. +function parseTime(str) { + const strTimeMatch = str.match(/(-?\d+):(\d+)/); + if (!strTimeMatch) { + return NaN; + } + let [, m, s] = strTimeMatch; + s = parseInt(s); + let t = parseInt(m) * 60; + t += t > 0 ? s : -s; + return t; +} + +// Convert number of seconds to duration string in h:mm:ss format. +function timeToDuration(s) { + let h = 0; + let m = Math.floor(s / 60); + s %= 60; + if (m > 59) { + h = Math.floor(m / 60); + m %= 60; + } + m = zeroPad(m, 2); + s = zeroPad(s, 2); + return h > 0 ? `${h}:${m}:${s}` : `${m}:${s}`; +} + +// Zero-pad an integer up to length. +function zeroPad(num, length) { + return ZU.lpad(`${num}`, "0", length); +} + +// Find the name of the guest in Lewis Lapham's podcast episode. +// NOTE: Usually the title is the guest's name, but not always. +// See: https://www.laphamsquarterly.org/content/vicars-christ, where the title +// is "Vicars of Christ". +// Therefore we try to infer the name using heuristics: +// 1. "The name often appears in the main-content paragraphs containing the +// words '[Lewis H.] Lapham (verb, speaks/talks) with [title] NAME [punct or +// 'about', but also possibly more noisy words]" +// 2. "It is very likely to be in the title." +// 3. "But if not 2, the name may also appear in the audio source file's name +// in the URL." +function inferEiCPodGuest(doc, headingText, epURL) { + // Take the basename of the episode audio without the last file extension. + // .../url/path/to/(basename).ext?query#frag + let [, epSource] = epURL.match(/^(?:.+\/)(.+)(?:\..+)$/); + + // Try to find name candidate by parsing the paragraph text. + const paragraphs = doc.querySelectorAll(".jp-jplayer ~ p"); + + let nameCandidate; + if (paragraphs) { + const textString = Array.from(paragraphs) + .map(x => ZU.trimInternal(x.textContent.trim())) + .join(" "); + // [Lewis[ H.] ]Lapham [verb] with (noisy name candidate)[, or about ] + // Note here "noisy name candidate" matches leniently, but + // non-greedily. Otherwise the group will match all the way to the last + // comma punct or "about". + const nameMatch = textString + .match(/(?:Lewis(?: H\.)? )Lapham \S+ with ([\S ]+?)(?:,| about)/); + if (nameMatch) { + nameCandidate = nameMatch[1]; + } + } + + // If no useful name candidate is extracted, here's our last ditch effort: + // fall back to using the episode file name alone (this is the case for + // some old episodes). + if (!nameCandidate) { + return epSource.replace(/_/g, " "); + } + + if (nameCandidate.toLowerCase().includes(headingText.toLowerCase())) { + // Name candidate (possibly surrounded by noise) is in title: This + // means the title can be taken to be the guest's name with high + // confidence. + return headingText; + } + else { + // Name candidate found but not in title. In this case, use to the + // audio file's name as a filter to clean up name candidate. + // So we need to case-normalize the episode filename. + epSource = epSource.toLowerCase(); + + // Generate "token stream" from name candidate, e.g. + // "Johann Sebastian Bach" -> ["Johann", "Sebastian", "Bach"] + // "Vita Sackerville-West" -> ["Vita", "Sackerville-West"] + const tokens = nameCandidate.split(" "); + const filteredTokens = []; // output + for (const token of tokens) { + // Token may contain punct such as period or comma as "noise" + // around the word, and apostrophe as internal "noise", but careful + // not to overgeneralize (TODO: replace any dash with one single + // minus-sign-hyphen (0x2D)?). Also, need to normalize and remove + // the diacritics. + const cleanToken + = token + .normalize("NFD") + .replace(/[\u0300-\u036F]/g, "") // Most of diacritics + .toLowerCase() + .split("") // to remove noisy puncts + .filter(x => !(x === "." || x === "," + || x === "'" || x === "’")) + .join(""); + + // Note that we use clean token for logic but original token + // for output. + if (epSource.includes(cleanToken)) { + filteredTokens.push(token); + } + } + return filteredTokens.join(" "); + } +} + +// Utility functions + +// Process author. Parse it as "[possibly Oxford] comma-separated, possibly +// with the word 'and'". +function parseAuthors(str, authorType = "author") { + return str.split(/(?:,|\s+and\s+)/) + .map(s => s.trim()) + .filter(Boolean) + .map(s => ZU.cleanAuthor(s, authorType)); +} + +// Convert from Roman numeral to integer. Note that the function assumes a +// correctly formed Roman numeral using letters up to C. +const ROMAN_NUMERAL = { + I: 1, + V: 5, + X: 10, + L: 50, + C: 100 // "D" and "M" unlikely to be encountered any time soon. +}; + +function romanToInt(str) { + return str.split("") + .map(i => ROMAN_NUMERAL[i.toUpperCase()]) + .reduce((sum, curValue, cur, arr) => { + const prev = cur - 1; + const trySum = sum + curValue; + if (cur > 0 && arr[prev] < curValue) { + // Should subtract instead of add. + return trySum - arr[prev] * 2; + } + return trySum; + }, 0); +} + +/** BEGIN TEST CASES **/ +var testCases = [ + { + "type": "web", + "url": "https://www.laphamsquarterly.org/search/node/mesopotamian", + "items": "multiple" + }, + { + "type": "web", + "url": "https://www.laphamsquarterly.org/education/schoolboy-where-are-you-going", + "items": [ + { + "itemType": "magazineArticle", + "title": "Schoolboy, Where Are You Going?", + "creators": [ + { + "firstName": "Moudhy", + "lastName": "Al-Rashid", + "creatorType": "author" + } + ], + "date": 2022, + "ISSN": "1935-7494", + "abstractNote": "Scribal education in the ancient Mesopotamian tablet house.", + "issue": 4, + "language": "en", + "libraryCatalog": "Lapham's Quarterly", + "publicationTitle": "Lapham’s Quarterly", + "url": "https://www.laphamsquarterly.org/education/schoolboy-where-are-you-going", + "volume": 14, + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.laphamsquarterly.org/roundtable/double-vision", + "items": [ + { + "itemType": "blogPost", + "title": "Double Vision", + "creators": [ + { + "firstName": "Frank", + "lastName": "Gonzalez-Crussi", + "creatorType": "author" + } + ], + "date": "2023-03-14T16:00:00.000Z", + "blogTitle": "Roundtable", + "language": "en", + "url": "https://www.laphamsquarterly.org/roundtable/double-vision", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.laphamsquarterly.org/roundtable/ancient-mesopotamian-tablet-cookbook", + "items": [ + { + "itemType": "blogPost", + "title": "The Ancient Mesopotamian Tablet as Cookbook", + "creators": [ + { + "firstName": "Gojko", + "lastName": "Barjamovic", + "creatorType": "author" + }, + { + "firstName": "Patricia Jurado", + "lastName": "Gonzalez", + "creatorType": "author" + }, + { + "firstName": "Chelsea A.", + "lastName": "Graham", + "creatorType": "author" + }, + { + "firstName": "Agnete W.", + "lastName": "Lassen", + "creatorType": "author" + }, + { + "firstName": "Nawal", + "lastName": "Nasrallah", + "creatorType": "author" + }, + { + "firstName": "Pia M.", + "lastName": "Sörensen", + "creatorType": "author" + } + ], + "date": "2019-06-10T16:00:00.000Z", + "blogTitle": "Roundtable", + "language": "en", + "url": "https://www.laphamsquarterly.org/roundtable/ancient-mesopotamian-tablet-cookbook", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.laphamsquarterly.org/content/peter-s-goodman", + "items": [ + { + "itemType": "podcast", + "title": "Peter S. Goodman", + "creators": [ + { + "firstName": "Lewis H.", + "lastName": "Lapham", + "creatorType": "author" + }, + { + "firstName": "Peter S.", + "lastName": "Goodman", + "creatorType": "guest" + } + ], + "abstractNote": "“Davos Man’s domination of the gains of globalization,” journalist Peter S. Goodman writes in Davos Man: How the Billionaires Devoured the World, “is how the United States found itself led by a patently unqualified casino developer as it grappled with a public health emergency that killed more Americans than those who died in World War I, World War II, and the Vietnam War", + "audioFileType": "audio/mpeg", + "episodeNumber": 87, + "language": "en", + "runningTime": "35:51", + "seriesTitle": "The World in Time", + "url": "https://www.laphamsquarterly.org/content/peter-s-goodman", + "attachments": [ + { + "title": "Audio", + "mimeType": "audio/mpeg" + }, + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.laphamsquarterly.org/content/soviets-spies", + "items": [ + { + "itemType": "podcast", + "title": "Soviets & Spies", + "creators": [], + "abstractNote": "Did an Englishman assist in the murder of Rasputin? Did a man knowns as the “Ace of Spies” almost carry off the assassination of the entire Bolshevik power structure? Did British agents really use semen as an invisible ink? Giles Milton, author of Russian Roulette: How British Spies Thwarted Lenin's Plot for Global Revolution, has the answers.", + "audioFileType": "audio/mpeg", + "episodeNumber": 61, + "language": "en", + "runningTime": "43:42", + "seriesTitle": "LQ Podcast", + "url": "https://www.laphamsquarterly.org/content/soviets-spies", + "attachments": [ + { + "title": "Audio", + "mimeType": "audio/mpeg" + }, + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.laphamsquarterly.org/content/paradise-city", + "items": [ + { + "itemType": "podcast", + "title": "To the Paradise City", + "creators": [ + { + "firstName": "Lewis H.", + "lastName": "Lapham", + "creatorType": "author" + }, + { + "firstName": "Brook", + "lastName": "Wilensky-Lanford", + "creatorType": "guest" + } + ], + "abstractNote": "Lewis Lapham talks with author Brook Wilensky-Lanford about the search for Adam and Eve’s hometown.", + "audioFileType": "audio/mpeg", + "language": "en", + "runningTime": "16:27", + "seriesTitle": "The World in Time", + "url": "https://www.laphamsquarterly.org/content/paradise-city", + "attachments": [ + { + "title": "Audio", + "mimeType": "audio/mpeg" + }, + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.laphamsquarterly.org/content/death-nothing-us", + "items": [ + { + "itemType": "podcast", + "title": "Death Is Nothing to Us", + "creators": [ + { + "firstName": "Lewis H.", + "lastName": "Lapham", + "creatorType": "author" + }, + { + "firstName": "Stephen", + "lastName": "Greenblatt", + "creatorType": "guest" + } + ], + "abstractNote": "Historian Stephen Greenblatt writes of “the concentrated force of the buried past” in The Swerve, his 2011 National Book Award winner in nonfiction.", + "audioFileType": "audio/mpeg", + "language": "en", + "runningTime": "20:05", + "seriesTitle": "The World in Time", + "url": "https://www.laphamsquarterly.org/content/death-nothing-us", + "attachments": [ + { + "title": "Audio", + "mimeType": "audio/mpeg" + }, + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.laphamsquarterly.org/content/roosevelt-montas", + "items": [ + { + "itemType": "podcast", + "title": "Roosevelt Montás", + "creators": [ + { + "firstName": "Lewis H.", + "lastName": "Lapham", + "creatorType": "author" + }, + { + "firstName": "Roosevelt", + "lastName": "Montás", + "creatorType": "guest" + } + ], + "abstractNote": "“In my sophomore year of high school, I came upon a remarkable book in a garbage pile next to the house where we rented an apartment in Queens,” scholar Roosevelt Montás writes at the beginning of Rescuing Socrates: How the Great Books Changed My Life and Why They Matter for a New Generation. “It was the second volume of the pretentiously bound Harvard Classics series, and it", + "audioFileType": "audio/mpeg", + "episodeNumber": 85, + "language": "en", + "runningTime": "32:19", + "seriesTitle": "The World in Time", + "url": "https://www.laphamsquarterly.org/content/roosevelt-montas", + "attachments": [ + { + "title": "Audio", + "mimeType": "audio/mpeg" + }, + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.laphamsquarterly.org/content/andrew-j-oshaughnessy", + "items": [ + { + "itemType": "podcast", + "title": "Andrew J. O’Shaughnessy", + "creators": [ + { + "firstName": "Lewis H.", + "lastName": "Lapham", + "creatorType": "author" + }, + { + "firstName": "Andrew J.", + "lastName": "O’Shaughnessy", + "creatorType": "guest" + } + ], + "abstractNote": "“Existing biographies of Thomas Jefferson,” the historian Andrew J. O’Shaughnessy writes in The Illimitable Freedom of the Human Mind: Thomas Jefferson’s Idea of a University, treat the retired president’s singular founding of a university “as merely an epilogue, while institutional histories give little consideration to the biographical context…Beginning at the age of", + "audioFileType": "audio/mpeg", + "episodeNumber": 84, + "language": "en", + "runningTime": "32:41", + "seriesTitle": "The World in Time", + "url": "https://www.laphamsquarterly.org/content/andrew-j-oshaughnessy", + "attachments": [ + { + "title": "Audio", + "mimeType": "audio/mpeg" + }, + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.laphamsquarterly.org/freedom/andrey-kurkov-picks-his-pen", + "items": [ + { + "itemType": "magazineArticle", + "title": "Andrey Kurkov Picks Up His Pen", + "creators": [ + { + "firstName": "Andrey", + "lastName": "Kurkov", + "creatorType": "author" + } + ], + "date": 2023, + "ISSN": "1935-7494", + "abstractNote": "On the freedom to write in Ukraine.", + "issue": 1, + "language": "en", + "libraryCatalog": "Lapham's Quarterly", + "publicationTitle": "Lapham’s Quarterly", + "rights": "Copyright © 2022 by Andrey Kurkov.", + "url": "https://www.laphamsquarterly.org/freedom/andrey-kurkov-picks-his-pen", + "volume": 15, + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [ + "From a speech delivered at the PEN World Voices Festival. The son of a doctor and a pilot, Kurkov trained as a Japanese translator and began writing novels while serving as a prison guard in Odesa. His novel Grey Bees, which he wrote after meeting refugees in Kyiv who made regular trips to the Donbas to deliver medicine, depicts the 2014 war through the perspective of a beekeeper. “For Ukrainians, freedom is more important than stability,” Kurkov said in a March 2022 interview. “For Russians, it is the opposite. Ukrainians change their presidents at each election, Russians keep their tsar until the tsar is dead.”" + ], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.laphamsquarterly.org/freedom/we-refuse-logic", + "items": [ + { + "itemType": "magazineArticle", + "title": "We Refuse This Logic", + "creators": [ + { + "firstName": "Arlen", + "lastName": "Austin", + "creatorType": "translator" + } + ], + "date": 2023, + "ISSN": "1935-7494", + "abstractNote": "The problem is not abortion.", + "issue": 1, + "language": "en", + "libraryCatalog": "Lapham's Quarterly", + "publicationTitle": "Lapham’s Quarterly", + "rights": "Translation copyright © 2022 by Arlen Austin.", + "url": "https://www.laphamsquarterly.org/freedom/we-refuse-logic", + "volume": 15, + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [ + "Movimento di Lotta Femminile di Padova, from “Pregnancy and Abortion.” In June 1971 Mariarosa Dalla Costa, who had been active in the Italian workers’ movement, convened a meeting in Padua to discuss demanding wages for housework. The meeting led to the formation of what came to be called Lotta Femminista, which produced pamphlets, conducted studies, and documented its militant activity. This manifesto was later published in Dalla Costa and Selma James’ book The Power of Women and the Subversion of the Community." + ], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.laphamsquarterly.org/youth/sweet-and-cold", + "items": [ + { + "itemType": "magazineArticle", + "title": "Sweet and Cold", + "creators": [ + { + "firstName": "Xu", + "lastName": "Wei", + "creatorType": "author" + }, + { + "firstName": "Jonathan", + "lastName": "Chaves", + "creatorType": "translator" + } + ], + "date": 2014, + "ISSN": "1935-7494", + "abstractNote": "What a shame that I have carried a boy, as he ate some candy, to his death.", + "issue": 3, + "language": "en", + "libraryCatalog": "Lapham's Quarterly", + "publicationTitle": "Lapham’s Quarterly", + "rights": "© 1986, Columbia University Press.", + "url": "https://www.laphamsquarterly.org/youth/sweet-and-cold", + "volume": 7, + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [ + "“A Kite.” After failing the civil-service examination on eight occasions, Xu became the personal secretary to a military commander in 1558 and assisted in defending his hometown from the attacks of Japanese pirates. After his patron’s downfall and death, he was faced with serious professional difficulties and, either insane or faking it effectively, attempted suicide by pushing an awl through his ear and pounding his testicles with a hammer. Later, he killed his third wife and went to prison but won release after seven years." + ], + "seeAlso": [] + } + ] + } +] +/** END TEST CASES **/ From 6634110def078862d2992c7c9835305a8f3cbe74 Mon Sep 17 00:00:00 2001 From: bfahrenfort <59982409+bfahrenfort@users.noreply.github.com> Date: Wed, 12 Apr 2023 05:24:30 -0500 Subject: [PATCH 012/125] Add Lexis+ Translator (#3012) --- Lexis+.js | 264 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 264 insertions(+) create mode 100644 Lexis+.js diff --git a/Lexis+.js b/Lexis+.js new file mode 100644 index 00000000000..f7357fffcf4 --- /dev/null +++ b/Lexis+.js @@ -0,0 +1,264 @@ +{ + "translatorID": "419638d9-9049-44ad-ba08-fa54ed24b5e6", + "label": "Lexis+", + "creator": "bfahrenfort", + "target": "^https?://plus\\.lexis\\..*/", + "minVersion": "5.0", + "maxVersion": "", + "priority": 100, + "inRepository": true, + "translatorType": 4, + "browserSupport": "gcsibv", + "lastUpdated": "2023-04-10 03:15:48" +} + +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2023 Brandon Fahrenfort + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +function detectWeb(doc, _url) { + if (doc.title.includes("results")) { + return "multiple"; + } + else if (/[a-zA-Z. ]+\s§\s\d+/.test(doc.title) + || /act/i.test(doc.title) + || /p\.l\./i.test(doc.title)) { // Match: ... Tex. Bus. & Com. Code § 26.01 ... + return "statute"; + } + else if (/\d+\s[a-zA-Z0-9. ]+\s\d+/.test(doc.title)) { // Match: ... 5 U.S. 137 ... + return "case"; + } + // TODO secondary sources + + return false; +} + +function getSearchResults(doc, url) { + var items = {}; + var nextTitle; + + if (detectWeb(doc, url) == "multiple") { + // TODO check what type of element it is (currently only working for 'cases' searches) + let titles = doc.querySelectorAll('a.titleLink'); + let dates = doc.querySelectorAll('span.metaDataItem'); // Not technically only dates, but that's all I use it for atm + var nextDate; + var dateOffset = 1; + + // dates[0] is first court name + nextDate = dates[dateOffset]; + dateOffset += 3; + // dates[2] is first citation + for (var i = 0; i < titles.length; i++) { + nextTitle = titles[i]; + items[nextTitle.href] = nextTitle.textContent + "(" + nextDate.textContent + ")"; + + // dates[0] is court name + nextDate = dates[dateOffset]; + + // dates[2] is a citation + } + + return items; + } + + return false; +} + +async function doWeb(doc, url) { + if (detectWeb(doc, url) == 'multiple') { + let items = await Zotero.selectItems(getSearchResults(doc, url)); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape(await requestDocument(url)); + } + } + else { + await scrape(doc, url); + } +} + +async function scrape(doc, url) { + var title = text(doc, 'h1#SS_DocumentTitle'); + + if (detectWeb(doc, url) == "case") { + var newCase = new Zotero.Item("case"); + //newCase.url = doc.location.href; // Disabled for style reasons + + newCase.title = title; + + newCase.notes.push({ note: "Snapshot: " + newCase.title + doc.getElementById('document-content').innerHTML }); + + let citation = text(doc, 'span.active-reporter'); + newCase.reporterVolume = citation.substring(0, citation.indexOf(' ')); + newCase.reporter = citation.substring(citation.indexOf(' ') + 1, citation.lastIndexOf(' ')); + newCase.firstPage = citation.substring(citation.lastIndexOf(' ') + 1); + + newCase.court = text(doc, 'p.SS_DocumentInfo', 0); + + newCase.dateDecided = text(doc, 'span.date'); + + let docket = text(doc, 'p.SS_DocumentInfo', 2); + if (/^no\./i.test(docket) + || /^\d+/.test(docket) + || /^case no\./i.test(docket)) { + newCase.docketNumber = docket; // This won't be in perfect cite form, shouldn't be a hassle unless you're citing dozens of memorandum opinions + } + + newCase.complete(); + } + else if (detectWeb(doc, url) == "statute") { + var newStatute = new Zotero.Item("statute"); + + //newStatute.url = doc.location.href; // Disabled for style reasons + + newStatute.title = title; + + newStatute.notes.push({ note: "Snapshot: " + newStatute.title + doc.getElementById('document-content').innerHTML }); + + let info = text(doc, 'p.SS_DocumentInfo'); + + let isolation = info.substring(info.search( + /\b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)/i + )); // isolate date on the frontend + newStatute.dateEnacted = isolation.substring(0, isolation.search(/[1-2][0-9][0-9][0-9]/) + 4); + + if (/act/i.test(title) + || /of\s[1-2][0-9][0-9][0-9]/i.test(title)) { // Session law, or act, not codified statute + // BB 21st ed. requires parallel cite to Pub. L. No. and Stat. for session laws + + // Title formatting + // TODO ZU's capitalizer is good, but doesn't work with closed-up abbreviations like P.L. or U.S. + // This could break titles in future. I do remove P.L. below though. + if (title === title.toUpperCase()) title = ZU.capitalizeTitle(title.toLowerCase(), true); // Some acts are capitalized + + // Remove some unnecessary information + var cleanedTitle = title; + let pLCite = title.match(/(\d+ p\.l\. \d+)/i); + let statCite = title.match(/(\d+ stat\. \d+)/i); + let enactedCite = title.match(/\d+ enacted [a-zA-Z0-9.]+ \d+/gi); + let part = title.match(/(part \d+(?: of \d+)?)/i); + if (pLCite) cleanedTitle = cleanedTitle.replace(pLCite[1], ''); + if (statCite) cleanedTitle = cleanedTitle.replace(statCite[1], ''); + if (part) cleanedTitle = cleanedTitle.replace(part[1], ''); + if (enactedCite) { + // Remove every enacted cite + for (var value of Object.values(enactedCite)) { + cleanedTitle = cleanedTitle.replace(value, ''); + } + } + cleanedTitle = cleanedTitle.replace(/(^\s*,)|(,\s*$)/g, ''); // Trim commas and whitespace + cleanedTitle = cleanedTitle.replace(/(^\s*,)|(,\s*$)/g, ''); // Another one + if (ZU.trim(cleanedTitle) === "") { // If the title's empty now, put it as the highest precedence citation in the title + if (pLCite) cleanedTitle = pLCite[1]; + else if (statCite) cleanedTitle = statCite[1]; + else if (enactedCite) { + cleanedTitle = enactedCite[0] + " & " + (Object.keys(enactedCite).length - 1) + " more"; + } + } + newStatute.title = cleanedTitle; + + // Reporter & citation formatting + var statutesAtLarge, publicLawNo; + let potentialReporter = text(doc, 'a.SS_ActiveRptr'); + if (potentialReporter) { // Sometimes Lexis is weird and doesn't give an ActiveRptr + if (/stat\./i.test(potentialReporter)) statutesAtLarge = potentialReporter; + else if (/pub\./i.test(potentialReporter) + || /p\.l\./i.test(potentialReporter)) { + publicLawNo = potentialReporter; + } + } + + let otherReporters = doc.querySelectorAll('span.SS_NonPaginatedRptr'); + + for (var i = 0; i < otherReporters.length; i++) { + var nextReporter = otherReporters[i].textContent; + if (/stat\./i.test(nextReporter)) statutesAtLarge = nextReporter; + else if (/pub\./i.test(nextReporter) + || /p\.l\./i.test(nextReporter)) { + publicLawNo = nextReporter; + } + } + + // Turn publicLawNo into the public law fields + if (/\d+-\d+/.test(publicLawNo)) { // Ex. P.L. 115-164 + let numPos = publicLawNo.search(/\d+-\d+/); + newStatute.publicLawNumber = publicLawNo.substring(numPos, publicLawNo.substring(numPos + 1).indexOf(' ')); // Gets 115-164 + + newStatute.session = newStatute.publicLawNumber.substring(0, newStatute.publicLawNumber.indexOf('-')); + } + else { // Ex. 115 P.L. 164 + let pLNumbers = publicLawNo.match(/(\d+) p\.l\. (\d+)/i); + newStatute.session = pLNumbers[1]; + newStatute.publicLawNumber = pLNumbers[1] + '-' + pLNumbers[2]; + } + + // Turn statutesAtLarge into the code#/code/section fields + // TODO in styles, check for "Stat." as the code, and if so, don't append a section symbol + let statNumbers = statutesAtLarge.match(/(\d+) stat\. (\d+)/i); + newStatute.codeNumber = statNumbers[1]; + newStatute.code = "Stat."; + newStatute.section = statNumbers[2]; + } + else { // Codified statute + // Title & citation formatting + if (title.match(/^\d+/)) { // Starts with digit, organized by title, ex. 47 U.S.C.S. § 230 + // Sadly, named groups aren't working + let groups = title.match(/^(\d+)\s([a-zA-Z0-9. ]+) § ([0-9.()a-zA-Z]+)/); + newStatute.codeNumber = groups[1]; + newStatute.code = groups[2]; + newStatute.section = groups[3]; + } + else { // Starts with letter, organized by code, ex. Tex. Bus. & Com. Code § 26.01 + let groups = title.match(/^([a-zA-Z&. ]+) § ([0-9.()a-zA-Z]+)/); + newStatute.code = groups[1]; + newStatute.section = groups[2]; + } + + // Reporter formatting, theoretically unnecessary but nice to have if it's there + /* + * Matches: + * P.L. 117-327 + * Pub. L. 117-327 + * Pub. Law 117-327 + * Pub. L. No. 117-327 + * Pub. Law No. 117-327 + * Public Law 117-327 + * Public Law Number 117-327 + * Public Law No. 117-327 + */ + let pL = info.match(/(p\.l\.|pub\. l(?:aw|\.)(?: no\.)?|public law(?: number| no\.)?)\s(\d+-\d+)/i); + if (pL) newStatute.publicLawNumber = pL[2]; + + if (newStatute.publicLawNumber) newStatute.session = newStatute.publicLawNumber.substring(0, newStatute.publicLawNumber.indexOf('-')); + } + + newStatute.notes.push({ note: "Document Info: " + info }); // Since the info section is all over the place, just dump the whole thing in for manual cite checks + + newStatute.complete(); + } +} + + +/** BEGIN TEST CASES **/ +var testCases = [ +] +/** END TEST CASES **/ From c878cdc1ea1148fa96daad3f0d2eaf630c029268 Mon Sep 17 00:00:00 2001 From: Dan Stillman Date: Sun, 16 Apr 2023 14:13:37 -0400 Subject: [PATCH 013/125] K10plus ISBN: Disable temporarily due to requests hanging https://forums.zotero.org/discussion/comment/432811/#Comment_432811 --- K10plus ISBN.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/K10plus ISBN.js b/K10plus ISBN.js index 0a6bf961c59..1e4c6182414 100644 --- a/K10plus ISBN.js +++ b/K10plus ISBN.js @@ -8,7 +8,7 @@ "priority": 99, "inRepository": true, "translatorType": 8, - "lastUpdated": "2022-06-10 06:34:45" + "lastUpdated": "2023-04-16 18:12:38" } /* @@ -35,7 +35,9 @@ along with Zotero. If not, see . */ function detectSearch(item) { - return !!item.ISBN; + // Disabled due to API outage + return false; + //return !!item.ISBN; } function doSearch(item) { From 618e4ed6933d6758acf09a0b404ac04785bf5212 Mon Sep 17 00:00:00 2001 From: Dan Stillman Date: Mon, 17 Apr 2023 06:54:48 -0400 Subject: [PATCH 014/125] Restore K10plus ISBN Closes #3019 --- K10plus ISBN.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/K10plus ISBN.js b/K10plus ISBN.js index 1e4c6182414..c738c3f05fb 100644 --- a/K10plus ISBN.js +++ b/K10plus ISBN.js @@ -8,7 +8,7 @@ "priority": 99, "inRepository": true, "translatorType": 8, - "lastUpdated": "2023-04-16 18:12:38" + "lastUpdated": "2023-04-17 10:53:59" } /* @@ -35,9 +35,7 @@ along with Zotero. If not, see . */ function detectSearch(item) { - // Disabled due to API outage - return false; - //return !!item.ISBN; + return !!item.ISBN; } function doSearch(item) { From b4829b4bfa2c6225a362ffb2930d8716a0ede312 Mon Sep 17 00:00:00 2001 From: Sebastian Karcher Date: Mon, 17 Apr 2023 10:05:08 -0400 Subject: [PATCH 015/125] Add CERN Document Server (#3016) --- CERN Document Server.js | 277 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 277 insertions(+) create mode 100644 CERN Document Server.js diff --git a/CERN Document Server.js b/CERN Document Server.js new file mode 100644 index 00000000000..c3b60300aa0 --- /dev/null +++ b/CERN Document Server.js @@ -0,0 +1,277 @@ +{ + "translatorID": "e4b51f32-bb3f-4d37-a46d-083efe534233", + "label": "CERN Document Server", + "creator": "Sebastian Karcher", + "target": "^https?://cds\\.cern\\.ch/(search\\?|collection/|record/)", + "minVersion": "5.0", + "maxVersion": "", + "priority": 100, + "inRepository": true, + "translatorType": 4, + "browserSupport": "gcsibv", + "lastUpdated": "2023-04-13 01:12:36" +} + +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2023 Sebastian Karcher + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + + +function detectWeb(doc, url) { + if (url.includes('/record/')) { + return getItemType(doc); + } + else if (getSearchResults(doc, true)) { + return 'multiple'; + } + return false; +} + +function getItemType(doc) { + var type = text(doc, '.formatRecordHeader').trim(); + // These are the most important one, but could probably be expanded further + switch (type) { + case "Article": + return "journalArticle"; + case "Thesis": + return "thesis"; + case "Report": + return "report"; + case "Book": + case "Books": + return "book"; + case "Preprint": + return "preprint"; + case "Talks": + return "presentation"; + default: + return "document"; + } +} +function getSearchResults(doc, checkOnly) { + var items = {}; + var found = false; + var rows = doc.querySelectorAll('strong>a.titlelink'); + for (let row of rows) { + let href = row.href; + let title = ZU.trimInternal(row.textContent); + if (!href || !title) continue; + if (checkOnly) return true; + found = true; + items[href] = title; + } + return found ? items : false; +} + +async function doWeb(doc, url) { + if (detectWeb(doc, url) == 'multiple') { + let items = await Zotero.selectItems(getSearchResults(doc, false)); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape(await requestDocument(url)); + } + } + else { + await scrape(doc, url); + } +} + +async function scrape(doc, url = doc.location.href) { + var pdfUrl = attr(doc, '#detailedrecordminipanelfile a[href*=".pdf"]', 'href'); + var abstract = attr(doc, 'meta[property="og:description"]', 'content'); + var thesisUniversity = attr(doc, 'meta[name="citation_dissertation_institution"]', 'content'); + // Z.debug(pdfUrl); + let bibUrl = url.replace(/[#?].*/, "") + '/export/hx?ln=en'; + let bibText = await requestText(bibUrl); + bibText = bibText.match(/
([\s\S]+?)<\/pre>/)[1];
+	// Z.debug(bibText)
+	let translator = Zotero.loadTranslator("import");
+	translator.setTranslator('9cb70025-a888-4a29-a210-93ec52da40d4');
+	translator.setString(bibText);
+	translator.setHandler('itemDone', (_obj, item) => {
+		if (pdfUrl) {
+			item.attachments.push({ url: pdfUrl, title: "Full Text PDF", mimeType: "application/pdf" });
+		}
+		delete item.itemID;
+		if (item.itemType == "thesis" && !item.university) {
+			item.university = thesisUniversity;
+		}
+		item.abstractNote = abstract;
+		item.itemType = getItemType(doc);
+		item.extra = "";
+		item.complete();
+	});
+	await translator.translate();
+}
+
+/** BEGIN TEST CASES **/
+var testCases = [
+	{
+		"type": "web",
+		"url": "https://cds.cern.ch/search?ln=en&sc=1&p=testing&action_search=Search&op1=a&m1=a&p1=&f1=&c=Articles+%26+Preprints&c=Books+%26+Proceedings&c=Presentations+%26+Talks&c=Periodicals+%26+Progress+Reports&c=Multimedia+%26+Outreach&c=International+Collaborations",
+		"detectedItemType": "multiple",
+		"items": "multiple"
+	},
+	{
+		"type": "web",
+		"url": "https://cds.cern.ch/record/2855572?ln=en",
+		"detectedItemType": "preprint",
+		"items": [
+			{
+				"itemType": "preprint",
+				"title": "A short history of Internet protocols at CERN",
+				"creators": [
+					{
+						"firstName": "Ben",
+						"lastName": "Segal",
+						"creatorType": "author"
+					}
+				],
+				"date": "1995",
+				"libraryCatalog": "CERN Document Server",
+				"place": "Geneva",
+				"repository": "CERN",
+				"url": "https://cds.cern.ch/record/2855572",
+				"attachments": [
+					{
+						"title": "Full Text PDF",
+						"mimeType": "application/pdf"
+					}
+				],
+				"tags": [],
+				"notes": [],
+				"seeAlso": []
+			}
+		]
+	},
+	{
+		"type": "web",
+		"url": "https://cds.cern.ch/collection/Published%20Articles?ln=en",
+		"detectedItemType": "multiple",
+		"items": "multiple"
+	},
+	{
+		"type": "web",
+		"url": "https://cds.cern.ch/record/2855446?ln=de",
+		"detectedItemType": "journalArticle",
+		"items": [
+			{
+				"itemType": "journalArticle",
+				"title": "Production of Σ⁰ Hyperon and Search of Σ⁰ Hypernuclei at LHC with ALICE",
+				"creators": [
+					{
+						"firstName": "Alexander",
+						"lastName": "Borissov",
+						"creatorType": "author"
+					},
+					{
+						"firstName": "Sergei",
+						"lastName": "Solokhin",
+						"creatorType": "author"
+					}
+				],
+				"date": "2023",
+				"DOI": "10.1134/S1063778823010131",
+				"abstractNote": "The first measurements of the transverse momentum ($p_{{{T}}}$) spectra and integrated yields and mean $p_{{T}}$ of $\\Sigma^{0}$ and $\\overline{\\Sigma}^{0}$ hyperons in $pp$ collisions at $\\sqrt{s}=7$ TeV at the LHC are presented. The $\\Sigma^{0}$($\\overline{\\Sigma}^{0}$) is reconstructed via its electromagnetic decay channel $\\Lambda$($\\overline{\\Lambda})+\\gamma$. The $\\Lambda$ ($\\overline{\\Lambda}$) baryon is reconstructed via its decay into $p+\\pi^{-}$ ($\\overline{{{p}}}+\\pi^{+}$), while the photon is detected by exploiting the unique capability of the ALICE detector to measure low-energy photons via conversion into $e^{+}e^{-}$ pairs in the detector material. The yield of $\\Sigma^{0}$ is compared to that of the $\\Lambda$ baryon, which has the same quark content but different isospin. These data contribute to the understanding of hadron production mechanisms and provide a reference for constraining QCD-inspired models and tuning Monte Carlo event generators such as PYTHIA. In addition, the feasibility of a search for a bound state of proton, neutron and $\\Sigma^{0}$($\\Sigma^{0}$ hypernuclei ${}^{3}_{\\Sigma^{0}}$H) is presented, based on the luminosities foreseen for the LHC Runs 3 and 4.",
+				"issue": "6",
+				"libraryCatalog": "CERN Document Server",
+				"pages": "970-975",
+				"publicationTitle": "Phys. At. Nucl.",
+				"url": "https://cds.cern.ch/record/2855446",
+				"volume": "85",
+				"attachments": [],
+				"tags": [],
+				"notes": [],
+				"seeAlso": []
+			}
+		]
+	},
+	{
+		"type": "web",
+		"url": "https://cds.cern.ch/record/2854594?ln=en",
+		"detectedItemType": "report",
+		"items": [
+			{
+				"itemType": "report",
+				"title": "Notes on the machine studies team informal seminars on transitional phenomena",
+				"creators": [
+					{
+						"firstName": "O",
+						"lastName": "Barbalat",
+						"creatorType": "author"
+					}
+				],
+				"date": "1968",
+				"institution": "CERN",
+				"libraryCatalog": "CERN Document Server",
+				"place": "Geneva",
+				"url": "https://cds.cern.ch/record/2854594",
+				"attachments": [
+					{
+						"title": "Full Text PDF",
+						"mimeType": "application/pdf"
+					}
+				],
+				"tags": [],
+				"notes": [],
+				"seeAlso": []
+			}
+		]
+	},
+	{
+		"type": "web",
+		"url": "https://cds.cern.ch/record/2854931?ln=en",
+		"detectedItemType": "thesis",
+		"items": [
+			{
+				"itemType": "thesis",
+				"title": "Pathlength-dependent jet quenching in the quark–gluon plasma at ALICE",
+				"creators": [
+					{
+						"firstName": "Caitlin",
+						"lastName": "Beattie",
+						"creatorType": "author"
+					}
+				],
+				"date": "2023",
+				"abstractNote": "At extremely high temperatures, the quarks and gluons that compose the fundamental building blocks of our universe undergo a phase transition from stable hadronic matter to become a deconfined quark--gluon plasma (QGP). One way to study this medium is through collisions of heavy ions, where extraordinarily high energy densities produce just such a deconfined state. Of particular interest are jets, collimated showers of hadrons that originate early in the collision and undergo modification as they traverse the QGP, thus probing the medium's properties and enabling the study of quantum chromodynamics at multiple scales. Notably, jets lose energy as they propagate through the medium, the pathlength dependence of which remains an open question. The answer is of significant interest, however, given that quantitative constraints on this dependence are closely related to the underlying mechanisms that drive jet quenching phenomena. This thesis will discuss the first measurement of jets using a technique known as event-shape engineering (ESE), a measurement made in an effort to constrain the pathlength dependence of jet energy loss. For this thesis, charged jets were measured in Pb--Pb collisions using the ALICE detector at the CERN Large Hadron Collider. These jets were then classified according to their angle with respect to the event plane, as well as the shape of the event that they traversed. No sensitivity of the jet spectra to the event shape was observed; however, the yields were seen to be dependent on the event-plane angle. Moreover, this dependence was stronger for highly-elliptical events and weaker for highly-isotropic events. Such results are consistent with descriptions of pathlength distributions that were studied in Trajectum and the assumption that jets lose energy in a pathlength-dependent manner. Further theoretical models are required to extract quantitative constraints from this study.",
+				"libraryCatalog": "CERN Document Server",
+				"university": "Yale University (US)",
+				"url": "https://cds.cern.ch/record/2854931",
+				"attachments": [
+					{
+						"title": "Full Text PDF",
+						"mimeType": "application/pdf"
+					}
+				],
+				"tags": [],
+				"notes": [
+					{
+						"note": "

Presented 07 Mar 2023

" + } + ], + "seeAlso": [] + } + ] + } +] +/** END TEST CASES **/ From 6d165ddda073ebdb67bcc4e7d672f37669789d7f Mon Sep 17 00:00:00 2001 From: Brendan O'Connell <68507084+brendan-oconnell@users.noreply.github.com> Date: Wed, 19 Apr 2023 00:44:05 +0200 Subject: [PATCH 016/125] Cambridge Core: Remove footnote links from end of title (#3018) --- Cambridge Core.js | 85 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 83 insertions(+), 2 deletions(-) diff --git a/Cambridge Core.js b/Cambridge Core.js index ecb5b81463c..9b843d2815c 100644 --- a/Cambridge Core.js +++ b/Cambridge Core.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2021-10-09 00:36:31" + "lastUpdated": "2023-04-17 08:48:37" } /* @@ -53,7 +53,7 @@ function detectWeb(doc, url) { } else return "book"; } - + // now let's check for multiples again, just to be sure. this handles some // rare listing page URLs that might not be included in the multiples // regex above. @@ -156,6 +156,11 @@ function scrape(doc, url) { if (item.date.includes("undefined")) { item.date = attr('meta[name="citation_online_date"]', "content"); } + // remove asterisk or 1 at end of title, e.g. https://www.cambridge.org/core/journals/american-political-science-review/article/abs/violence-in-premodern-societies-rural-colombia/A14B0BB4130A2BA6BE79E2853597526E + const titleElem = doc.querySelector("#maincontent h1"); + if (titleElem.querySelector('a:last-child')) { + item.title = titleElem.firstChild.textContent; + } item.complete(); }); @@ -453,6 +458,82 @@ var testCases = [ "type": "web", "url": "https://www.cambridge.org/core/journals/ajs-review/latest-issue", "items": "multiple" + }, + { + "type": "web", + "url": "https://www.cambridge.org/core/journals/american-political-science-review/article/abs/violence-in-premodern-societies-rural-colombia/A14B0BB4130A2BA6BE79E2853597526E", + "items": [ + { + "itemType": "journalArticle", + "title": "Violence in Pre-Modern Societies: Rural Colombia", + "creators": [ + { + "firstName": "Richard S.", + "lastName": "Weinert", + "creatorType": "author" + } + ], + "date": "1966/06", + "DOI": "10.2307/1953360", + "ISSN": "0003-0554, 1537-5943", + "abstractNote": "Violence is a common phenomenon in developing polities which has received little attention. Clearly a Peronist riot in Buenos Aires, a land invasion in Lima, and a massacre in rural Colombia are all different. Yet we have no typology which relates types of violence to stages or patterns of economic or social development. We know little of the causes, incidence or functions of different forms of violence. This article is an effort to understand one type of violence which can occur in societies in transition.Violence in Colombia has traditionally accompanied transfers of power at the national level. This can account for its outbreak in 1946, when the Conservative Party replaced the Liberals. It cannot account for the intensity or duration of rural violence for two decades. This article focuses primarily on the violence from 1946 to 1953, and explains its intensification and duration as the defense of a traditional sacred order against secular modernizing tendencies undermining that order. We shall discuss violence since 1953 in the concluding section.", + "issue": "2", + "language": "en", + "libraryCatalog": "Cambridge University Press", + "pages": "340-347", + "publicationTitle": "American Political Science Review", + "shortTitle": "Violence in Pre-Modern Societies", + "url": "https://www.cambridge.org/core/journals/american-political-science-review/article/abs/violence-in-premodern-societies-rural-colombia/A14B0BB4130A2BA6BE79E2853597526E", + "volume": "60", + "attachments": [ + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.cambridge.org/core/journals/journal-of-public-policy/article/abs/when-consumers-oppose-consumer-protection-the-politics-of-regulatory-backlash/2C8E6B9BB6881A233B8936D9AD2C6305", + "items": [ + { + "itemType": "journalArticle", + "title": "When Consumers Oppose Consumer Protection: The Politics of Regulatory Backlash", + "creators": [ + { + "firstName": "David", + "lastName": "Vogel", + "creatorType": "author" + } + ], + "date": "1990/10", + "DOI": "10.1017/S0143814X00006085", + "ISSN": "1469-7815, 0143-814X", + "abstractNote": "This article examines a neglected phenomenon in the existing literature on social regulation, namely political opposition to regulation that comes not from business but from consumers. It examines four cases of successful grass-roots consumer opposition to government health and safety regulations in the United States. Two involve rules issued by the National Highway Traffic Safety Administration, a 1974 requirement that all new automobiles be equipped with an engine-interlock system, and a 1967 rule that denied federal highway funds to states that did not require motorcyclists to wear a helmet. In 1977, Congress overturned the Food and Drug Administration's ban on the artificial sweetener, saccharin. Beginning in 1987, the FDA began to yield to pressures from the gay community by agreeing to streamline its procedures for the testing and approval of new drugs designed to fight AIDS and other fatal diseases. The article identifies what these regulations have in common and examines their significance for our understanding the politics of social regulation in the United States and other industrial nations.", + "issue": "4", + "language": "en", + "libraryCatalog": "Cambridge University Press", + "pages": "449-470", + "publicationTitle": "Journal of Public Policy", + "shortTitle": "When Consumers Oppose Consumer Protection", + "url": "https://www.cambridge.org/core/journals/journal-of-public-policy/article/abs/when-consumers-oppose-consumer-protection-the-politics-of-regulatory-backlash/2C8E6B9BB6881A233B8936D9AD2C6305", + "volume": "10", + "attachments": [ + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] } ] /** END TEST CASES **/ From 3bd1342daf10530a37e2634e83d2d1f3190601e7 Mon Sep 17 00:00:00 2001 From: Abe Jellinek Date: Wed, 19 Apr 2023 10:30:07 -0400 Subject: [PATCH 017/125] MARCXML: Throw on invalid XML, skip empty records --- MARCXML.js | 44 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/MARCXML.js b/MARCXML.js index 085ce47cc18..a443daf5201 100644 --- a/MARCXML.js +++ b/MARCXML.js @@ -8,10 +8,9 @@ "priority": 100, "inRepository": true, "translatorType": 1, - "lastUpdated": "2019-07-11 13:12:18" + "lastUpdated": "2023-04-19 14:30:00" } - /* ***** BEGIN LICENSE BLOCK ***** @@ -53,23 +52,22 @@ function detectImport() { function doImport() { - var text = ""; - var line; - while ((line = Zotero.read()) !== false) { - text += line; - } // call MARC translator var translator = Zotero.loadTranslator("import"); translator.setTranslator("a6ee60df-1ddc-4aae-bb25-45e0537be973"); translator.getTranslatorObject(function (marc) { - var parser = new DOMParser(); - var xml = parser.parseFromString(text, 'text/xml'); + var xml = Zotero.getXML(); // define the marc namespace var ns = { marc: "http://www.loc.gov/MARC21/slim" }; var records = ZU.xpath(xml, '//marc:record', ns); for (let rec of records) { + if (!ZU.xpath(rec, "./marc:datafield", ns).length) { + Zotero.debug('Skipping empty record'); + continue; + } + // create one new item per record var record = new marc.record(); var newItem = new Zotero.Item(); @@ -366,6 +364,34 @@ var testCases = [ "seeAlso": [] } ] + }, + { + "type": "import", + "input": "\n\nMARC21-xml\nxml\n\n\n00000pam a2200000 cc4500\n1242883924\nDE-101\n20230125220155.0\ntu\n211010s2023 gw ||||| |||| 00||||ger \n\n23,A04\n21,N41\ndnb\n\n\nDE-101\n1242883924\n\n\n9783525560624\nFesteinband : EUR 85.00 (DE), EUR 88.00 (AT)\n978-3-525-56062-4\n\n\n3525560621\n3-525-56062-1\n\n\n9783525560624\n\n\nBestellnummer: VUR0008785\n\n\n(DE-599)DNB1242883924\n\n\n(OCoLC)1365383776\n\n\n(OCoLC)1275381065\n\n\n1245\nger\nDE-101\n9999\nrda\n\n\nger\n\n\nXA-DE\n\n\n230\n943\nDE-101\n23sdnb\n\n\n(DE-588)111415403\nhttps://d-nb.info/gnd/111415403\n(DE-101)111415403\nErhart, Hannelore\n1927-2013\nVerfasser\naut\ngnd\n\n\nKatharina Staritz\nBand 2.\n\n1903-1953 / Ilse Meseberg-Haubold, Dietgard Meyer ; unter Mitarbeit von Hannelore Erhart †\n\n\nHannelore Erhart ; Ilse Meseberg-Haubold ; Dietgard Meyer. Mit einem Exkurs Elisabeth Schmitz\n\n\n\nGöttingen\nVandenhoeck & Ruprecht\n[2023]\n\n\nXV, 629 Seiten\nIllustrationen\n1094 g\n\n\nText\ntxt\nrdacontent\n\n\nohne Hilfsmittel zu benutzen\nn\nrdamedia\n\n\nBand\nnc\nrdacarrier\n\n\n\n2\n", + "items": [ + { + "itemType": "book", + "title": "Katharina Staritz. Band 2: 1903-1953 / Ilse Meseberg-Haubold, Dietgard Meyer ; unter Mitarbeit von Hannelore Erhart †", + "creators": [ + { + "firstName": "Hannelore", + "lastName": "Erhart", + "creatorType": "author" + } + ], + "date": "2023", + "ISBN": "9783525560624 3525560621", + "callNumber": "230 943", + "language": "ger", + "numPages": "629", + "place": "Göttingen", + "publisher": "Vandenhoeck & Ruprecht", + "attachments": [], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] } ] /** END TEST CASES **/ From ed47524212070bfd174ebee047176ebb01cbec04 Mon Sep 17 00:00:00 2001 From: Sebastian Karcher Date: Wed, 19 Apr 2023 17:09:56 -0400 Subject: [PATCH 018/125] Update Zenodo for Dataset item type (#3021) --- Zenodo.js | 141 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 78 insertions(+), 63 deletions(-) diff --git a/Zenodo.js b/Zenodo.js index 6712de9e500..d9026151222 100644 --- a/Zenodo.js +++ b/Zenodo.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2021-06-07 05:21:23" + "lastUpdated": "2023-04-19 20:55:45" } /* @@ -35,59 +35,65 @@ ***** END LICENSE BLOCK ***** */ +const datasetType = ZU.fieldIsValidForType('title', 'dataset') + ? 'dataset' + : 'document'; + function detectWeb(doc, url) { if (url.includes('/record/')) { var collections = ZU.xpath(doc, '//span[@class="pull-right"]/span[contains(@class, "label-default")]'); - for (var i=0; i Date: Thu, 20 Apr 2023 18:23:06 +0200 Subject: [PATCH 019/125] dblp translator: fixes selector and some typos (#3023) --- DBLP Computer Science Bibliography.js | 68 +++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 4 deletions(-) diff --git a/DBLP Computer Science Bibliography.js b/DBLP Computer Science Bibliography.js index eb345f2f907..132e55913da 100644 --- a/DBLP Computer Science Bibliography.js +++ b/DBLP Computer Science Bibliography.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2021-09-20 19:47:25" + "lastUpdated": "2023-04-20 13:37:43" } /* @@ -64,7 +64,7 @@ function detectWeb(doc, url) { function scrape(doc, _url) { - let allData = doc.querySelectorAll('#bibtex-section'); + let allData = doc.querySelectorAll('#bibtex-section > pre'); let firstData = allData[0]; var firstDataText = firstData.textContent.replace(/ ee\s*=/, " url ="); // e.g. ee = {http://dx.doi.org/10.1007/978-3-319-00035-0_37}, @@ -103,7 +103,7 @@ function scrapeMainPart(firstDataText, secondDataItem) { trans.setHandler('itemDone', function (obj, item) { if (secondDataItem) { if (secondDataItem.title && item.itemType == "conferencePaper") item.proceedingsTitle = secondDataItem.title; - if (secondDataItem.title && item.itemType == "bookSection") item.booktitle = secondDataItem.titel; + if (secondDataItem.title && item.itemType == "bookSection") item.bookTitle = secondDataItem.title; if (secondDataItem.creators && secondDataItem.creators.length > 0) item.creators = item.creators.concat(secondDataItem.creators); if (secondDataItem.publisher && !item.publisher) item.publisher = secondDataItem.publisher; if (secondDataItem.series && !item.series) item.series = secondDataItem.series; @@ -115,7 +115,7 @@ function scrapeMainPart(firstDataText, secondDataItem) { // yet contain a doi, then save the doi and delete the url. // If the item contains the doi corresponding to the url // then just delete the url and keep the doi. - if (item.url && item.url.search(/^https?:\/\/(?:dx\.)?doi\.org\/10\./i) != -1) { + if (item.url && /^https?:\/\/(?:dx\.)?doi\.org\/10\./i.test(item.url)) { var doi = ZU.cleanDOI(item.url); if (doi && (!item.DOI || item.DOI == doi)) { item.DOI = doi; @@ -397,6 +397,66 @@ var testCases = [ "seeAlso": [] } ] + }, + { + "type": "web", + "url": "https://dblp.org/rec/reference/choice/LangX16.html?view=bibtex¶m=2", + "detectedItemType": "bookSection", + "items": [ + { + "itemType": "bookSection", + "title": "Voting in Combinatorial Domains", + "creators": [ + { + "firstName": "Jérôme", + "lastName": "Lang", + "creatorType": "author" + }, + { + "firstName": "Lirong", + "lastName": "Xia", + "creatorType": "author" + }, + { + "firstName": "Felix", + "lastName": "Brandt", + "creatorType": "editor" + }, + { + "firstName": "Vincent", + "lastName": "Conitzer", + "creatorType": "editor" + }, + { + "firstName": "Ulle", + "lastName": "Endriss", + "creatorType": "editor" + }, + { + "firstName": "Jérôme", + "lastName": "Lang", + "creatorType": "editor" + }, + { + "firstName": "Ariel D.", + "lastName": "Procaccia", + "creatorType": "editor" + } + ], + "date": "2016", + "ISBN": "9781107446984", + "bookTitle": "Handbook of Computational Social Choice", + "extra": "DOI: 10.1017/CBO9781107446984.010", + "itemID": "DBLP:reference/choice/LangX16", + "libraryCatalog": "DBLP Computer Science Bibliography", + "pages": "197–222", + "publisher": "Cambridge University Press", + "attachments": [], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] } ] /** END TEST CASES **/ From 9d1a05070bac47c91e1e73ccae371f0f32b7a0ac Mon Sep 17 00:00:00 2001 From: Abe Jellinek Date: Fri, 21 Apr 2023 12:06:27 -0400 Subject: [PATCH 020/125] CI: Upgrade dependencies --- .../bin/teslint.js | 6 +- .../package-lock.json | 151 ++++++ .../package.json | 2 +- package-lock.json | 430 +++++++++--------- package.json | 3 +- 5 files changed, 359 insertions(+), 233 deletions(-) create mode 100644 .ci/eslint-plugin-zotero-translator/package-lock.json diff --git a/.ci/eslint-plugin-zotero-translator/bin/teslint.js b/.ci/eslint-plugin-zotero-translator/bin/teslint.js index e8286b462c2..dc5680fd2a0 100755 --- a/.ci/eslint-plugin-zotero-translator/bin/teslint.js +++ b/.ci/eslint-plugin-zotero-translator/bin/teslint.js @@ -4,7 +4,7 @@ const fs = require('fs'); const path = require('path'); -const find = require('recursive-readdir-synchronous'); +const find = require('recursive-readdir'); const { ESLint } = require("eslint"); const argv = require('commander'); @@ -35,7 +35,7 @@ async function main() { let allResults = []; function findIgnore(file, stats) { - if (stats.isDirectory()) return (path.basename(file) == "node_modules"); + if (stats.isDirectory()) return (path.basename(file) == "node_modules" || path.basename(file) == ".ci"); return !file.endsWith('.js'); } for (const target of argv.args) { @@ -43,7 +43,7 @@ async function main() { console.error(`Target file '${target}' does not exist; skipping`); // eslint-disable-line no-console continue; } - const files = fs.lstatSync(target).isDirectory() ? find(target, [findIgnore]) : [target]; + const files = fs.lstatSync(target).isDirectory() ? await find(target, [findIgnore]) : [target]; for (const file of files) { if (path.dirname(path.resolve(file)) === translators.cache.repo) { const translator = translators.cache.get(file); diff --git a/.ci/eslint-plugin-zotero-translator/package-lock.json b/.ci/eslint-plugin-zotero-translator/package-lock.json new file mode 100644 index 00000000000..b0b16874ada --- /dev/null +++ b/.ci/eslint-plugin-zotero-translator/package-lock.json @@ -0,0 +1,151 @@ +{ + "name": "eslint-plugin-zotero-translator", + "version": "0.0.1", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "eslint-plugin-zotero-translator", + "version": "0.0.1", + "license": "ISC", + "dependencies": { + "commander": "^2.19.0", + "find-root": "^1.1.0", + "recursive-readdir": "^2.2.3", + "require-dir": "^1.2.0", + "uuid": "^3.3.2" + }, + "bin": { + "teslint": "bin/teslint.js" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" + }, + "node_modules/find-root": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/find-root/-/find-root-1.1.0.tgz", + "integrity": "sha512-NKfW6bec6GfKc0SGx1e07QZY9PE99u0Bft/0rzSD5k3sO/vwkVUpDUKVm5Gpp5Ue3YfShPFTX2070tDs5kB9Ng==" + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/recursive-readdir": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/recursive-readdir/-/recursive-readdir-2.2.3.tgz", + "integrity": "sha512-8HrF5ZsXk5FAH9dgsx3BlUer73nIhuj+9OrQwEbLTPOBzGkL1lsFCR01am+v+0m2Cmbs1nP12hLDl5FA7EszKA==", + "dependencies": { + "minimatch": "^3.0.5" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/require-dir": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/require-dir/-/require-dir-1.2.0.tgz", + "integrity": "sha512-LY85DTSu+heYgDqq/mK+7zFHWkttVNRXC9NKcKGyuGLdlsfbjEPrIEYdCVrx6hqnJb+xSu3Lzaoo8VnmOhhjNA==", + "engines": { + "node": "*" + } + }, + "node_modules/uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", + "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", + "bin": { + "uuid": "bin/uuid" + } + } + }, + "dependencies": { + "balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" + }, + "find-root": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/find-root/-/find-root-1.1.0.tgz", + "integrity": "sha512-NKfW6bec6GfKc0SGx1e07QZY9PE99u0Bft/0rzSD5k3sO/vwkVUpDUKVm5Gpp5Ue3YfShPFTX2070tDs5kB9Ng==" + }, + "minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "recursive-readdir": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/recursive-readdir/-/recursive-readdir-2.2.3.tgz", + "integrity": "sha512-8HrF5ZsXk5FAH9dgsx3BlUer73nIhuj+9OrQwEbLTPOBzGkL1lsFCR01am+v+0m2Cmbs1nP12hLDl5FA7EszKA==", + "requires": { + "minimatch": "^3.0.5" + } + }, + "require-dir": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/require-dir/-/require-dir-1.2.0.tgz", + "integrity": "sha512-LY85DTSu+heYgDqq/mK+7zFHWkttVNRXC9NKcKGyuGLdlsfbjEPrIEYdCVrx6hqnJb+xSu3Lzaoo8VnmOhhjNA==" + }, + "uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" + } + } +} diff --git a/.ci/eslint-plugin-zotero-translator/package.json b/.ci/eslint-plugin-zotero-translator/package.json index ec79ad4b0de..85cc8220cc5 100644 --- a/.ci/eslint-plugin-zotero-translator/package.json +++ b/.ci/eslint-plugin-zotero-translator/package.json @@ -14,7 +14,7 @@ "dependencies": { "commander": "^2.19.0", "find-root": "^1.1.0", - "recursive-readdir-synchronous": "0.0.4", + "recursive-readdir": "^2.2.3", "require-dir": "^1.2.0", "uuid": "^3.3.2" } diff --git a/package-lock.json b/package-lock.json index 8f49ab3e115..b8e7c8d2f33 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,9 +10,8 @@ "license": "CC0-1.0", "devDependencies": { "@zotero/eslint-config": "^1.0.5", - "chalk": "^4.1.0", "chromedriver": "^95.0.0", - "eslint": "^8.29.0", + "eslint": "^8.38.0", "eslint-plugin-zotero-translator": "file:.ci/eslint-plugin-zotero-translator", "selenium-webdriver": "^4.0.0-alpha.7" } @@ -24,7 +23,7 @@ "dependencies": { "commander": "^2.19.0", "find-root": "^1.1.0", - "recursive-readdir-synchronous": "0.0.4", + "recursive-readdir": "^2.2.3", "require-dir": "^1.2.0", "uuid": "^3.3.2" }, @@ -35,16 +34,40 @@ "node": ">=16.0.0" } }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", + "integrity": "sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==", + "dev": true, + "dependencies": { + "eslint-visitor-keys": "^3.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.5.0.tgz", + "integrity": "sha512-vITaYzIcNmjn5tF5uxcZ/ft7/RXGrMUIS9HalWckEOF6ESiwXKoMzAQf2UW0aVd6rnOeExTJVd5hmWXucBKGXQ==", + "dev": true, + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, "node_modules/@eslint/eslintrc": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.3.3.tgz", - "integrity": "sha512-uj3pT6Mg+3t39fvLrj8iuCIJ38zKO9FpGtJ4BBJebJhEwjoT+KLVNCcHT5QC9NGRIEi7fZ0ZR8YRb884auB4Lg==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.0.2.tgz", + "integrity": "sha512-3W4f5tDUra+pA+FzgugqL2pRimUTDJWKr7BINqOpkZrC0uYI0NIc0/JFgBROCU07HR6GieA5m3/rsPIhDmCXTQ==", "dev": true, "dependencies": { "ajv": "^6.12.4", "debug": "^4.3.2", - "espree": "^9.4.0", - "globals": "^13.15.0", + "espree": "^9.5.1", + "globals": "^13.19.0", "ignore": "^5.2.0", "import-fresh": "^3.2.1", "js-yaml": "^4.1.0", @@ -58,6 +81,15 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/@eslint/js": { + "version": "8.38.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.38.0.tgz", + "integrity": "sha512-IoD2MfUnOV58ghIHCiil01PcohxjbYR/qCxsoC+xNgUwh1EY8jOOrYmu3d3a71+tJJ23uscEV4X2HJWMsPJu4g==", + "dev": true, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.8", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.8.tgz", @@ -133,9 +165,9 @@ "dev": true }, "node_modules/@types/node": { - "version": "18.11.15", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.15.tgz", - "integrity": "sha512-VkhBbVo2+2oozlkdHXLrb3zjsRkpdnaU2bXmX8Wgle3PUi569eLRaHGlgETQHR7lLL1w7GiG3h9SnePhxNDecw==", + "version": "18.15.13", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.15.13.tgz", + "integrity": "sha512-N+0kuo9KgrUQ1Sn/ifDXsvg0TTleP7rIy4zOBGECxAljqvqfqpTfzx0Q1NUedOixRMBfe2Whhb056a42cWs26Q==", "dev": true, "optional": true }, @@ -159,9 +191,9 @@ } }, "node_modules/acorn": { - "version": "8.8.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.1.tgz", - "integrity": "sha512-7zFpHzhnqYKrkYdUjF1HI1bzd0VygEGX8lFk4k5zVMqHEoES+P+7TKI+EvLO9WVMJ8eekdO0aDEK044xTXwPPA==", + "version": "8.8.2", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.2.tgz", + "integrity": "sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw==", "dev": true, "bin": { "acorn": "bin/acorn" @@ -502,13 +534,16 @@ } }, "node_modules/eslint": { - "version": "8.29.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.29.0.tgz", - "integrity": "sha512-isQ4EEiyUjZFbEKvEGJKKGBwXtvXX+zJbkVKCgTuB9t/+jUBcy8avhkEwWJecI15BkRkOYmvIM5ynbhRjEkoeg==", + "version": "8.38.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.38.0.tgz", + "integrity": "sha512-pIdsD2jwlUGf/U38Jv97t8lq6HpaU/G9NKbYmpWpZGw3LdTNhZLbJePqxOXGB5+JEKfOPU/XLxYxFh03nr1KTg==", "dev": true, "dependencies": { - "@eslint/eslintrc": "^1.3.3", - "@humanwhocodes/config-array": "^0.11.6", + "@eslint-community/eslint-utils": "^4.2.0", + "@eslint-community/regexpp": "^4.4.0", + "@eslint/eslintrc": "^2.0.2", + "@eslint/js": "8.38.0", + "@humanwhocodes/config-array": "^0.11.8", "@humanwhocodes/module-importer": "^1.0.1", "@nodelib/fs.walk": "^1.2.8", "ajv": "^6.10.0", @@ -518,16 +553,15 @@ "doctrine": "^3.0.0", "escape-string-regexp": "^4.0.0", "eslint-scope": "^7.1.1", - "eslint-utils": "^3.0.0", - "eslint-visitor-keys": "^3.3.0", - "espree": "^9.4.0", - "esquery": "^1.4.0", + "eslint-visitor-keys": "^3.4.0", + "espree": "^9.5.1", + "esquery": "^1.4.2", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", "file-entry-cache": "^6.0.1", "find-up": "^5.0.0", "glob-parent": "^6.0.2", - "globals": "^13.15.0", + "globals": "^13.19.0", "grapheme-splitter": "^1.0.4", "ignore": "^5.2.0", "import-fresh": "^3.0.0", @@ -542,7 +576,6 @@ "minimatch": "^3.1.2", "natural-compare": "^1.4.0", "optionator": "^0.9.1", - "regexpp": "^3.2.0", "strip-ansi": "^6.0.1", "strip-json-comments": "^3.1.0", "text-table": "^0.2.0" @@ -562,9 +595,9 @@ "link": true }, "node_modules/eslint-scope": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.1.1.tgz", - "integrity": "sha512-QKQM/UXpIiHcLqJ5AOyIW7XZmzjkzQXYE54n1++wb0u9V/abW3l9uQnxX8Z5Xd18xyKIMTUAyQ0k1e8pz6LUrw==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.0.tgz", + "integrity": "sha512-DYj5deGlHBfMt15J7rdtyKNq/Nqlv5KfU4iodrQ019XESsRnwXH9KAE0y3cwtUHDo2ob7CypAnCqefh6vioWRw==", "dev": true, "dependencies": { "esrecurse": "^4.3.0", @@ -572,53 +605,32 @@ }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - } - }, - "node_modules/eslint-utils": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-3.0.0.tgz", - "integrity": "sha512-uuQC43IGctw68pJA1RgbQS8/NP7rch6Cwd4j3ZBtgo4/8Flj4eGE7ZYSZRN3iq5pVUv6GPdW5Z1RFleo84uLDA==", - "dev": true, - "dependencies": { - "eslint-visitor-keys": "^2.0.0" - }, - "engines": { - "node": "^10.0.0 || ^12.0.0 || >= 14.0.0" }, "funding": { - "url": "https://github.com/sponsors/mysticatea" - }, - "peerDependencies": { - "eslint": ">=5" - } - }, - "node_modules/eslint-utils/node_modules/eslint-visitor-keys": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz", - "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==", - "dev": true, - "engines": { - "node": ">=10" + "url": "https://opencollective.com/eslint" } }, "node_modules/eslint-visitor-keys": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.3.0.tgz", - "integrity": "sha512-mQ+suqKJVyeuwGYHAdjMFqjCyfl8+Ldnxuyp3ldiMBFKkvytrXUZWaiPCEav8qDHKty44bD+qV1IP4T+w+xXRA==", + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.0.tgz", + "integrity": "sha512-HPpKPUBQcAsZOsHAFwTtIKcYlCje62XB7SEAcxjtmW6TD1WVpkS6i6/hOVtTZIl4zGj/mBqpFVGvaDneik+VoQ==", "dev": true, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" } }, "node_modules/espree": { - "version": "9.4.1", - "resolved": "https://registry.npmjs.org/espree/-/espree-9.4.1.tgz", - "integrity": "sha512-XwctdmTO6SIvCzd9810yyNzIrOrqNYV9Koizx4C/mRhf9uq0o4yHoCEU/670pOxOL/MSraektvSAji79kX90Vg==", + "version": "9.5.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.5.1.tgz", + "integrity": "sha512-5yxtHSZXRSW5pvv3hAlXM5+/Oswi1AUFqBmbibKb5s6bp3rGIDkyXU6xCoyuuLhijr4SFwPrXRoZjz0AZDN9tg==", "dev": true, "dependencies": { "acorn": "^8.8.0", "acorn-jsx": "^5.3.2", - "eslint-visitor-keys": "^3.3.0" + "eslint-visitor-keys": "^3.4.0" }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" @@ -628,9 +640,9 @@ } }, "node_modules/esquery": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.0.tgz", - "integrity": "sha512-cCDispWt5vHHtwMY2YrAQ4ibFkAL8RbH5YGBnZBc90MolvvfkkQcJro/aZiAQUlQ3qgrYS6D6v8Gc5G5CQsc9w==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz", + "integrity": "sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==", "dev": true, "dependencies": { "estraverse": "^5.1.0" @@ -736,9 +748,9 @@ "dev": true }, "node_modules/fastq": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.14.0.tgz", - "integrity": "sha512-eR2D+V9/ExcbF9ls441yIuN6TI2ED1Y2ZcA5BmMtJsOkWOFRJQ0Jt0g1UwqXJJVAb+V+umH5Dfr8oh4EVP7VVg==", + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", + "integrity": "sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==", "dev": true, "dependencies": { "reusify": "^1.0.4" @@ -892,9 +904,9 @@ } }, "node_modules/globals": { - "version": "13.19.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.19.0.tgz", - "integrity": "sha512-dkQ957uSRWHw7CFXLUtUHQI3g3aWApYhfNR2O6jn/907riyTYKVBmxYVROkBcY614FSSeSJh7Xm7SrUWCxvJMQ==", + "version": "13.20.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.20.0.tgz", + "integrity": "sha512-Qg5QtVkCy/kv3FUSlu4ukeZDVf9ee0iXLAUYX13gbR17bnejFTzr4iS9bY7kwCf1NztRNm1t91fjOiyx4CSwPQ==", "dev": true, "dependencies": { "type-fest": "^0.20.2" @@ -927,9 +939,9 @@ } }, "node_modules/graceful-fs": { - "version": "4.2.10", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", - "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", "dev": true }, "node_modules/grapheme-splitter": { @@ -961,9 +973,9 @@ } }, "node_modules/ignore": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.1.tgz", - "integrity": "sha512-d2qQLzTJ9WxQftPAuEQpSPmKqzxePjzVbpAVv62AQ64NTL+wR4JkrVqR/LqFsFEUsHDAiId52mJteHDFuDkElA==", + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz", + "integrity": "sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ==", "dev": true, "engines": { "node": ">= 4" @@ -1115,9 +1127,9 @@ "dev": true }, "node_modules/js-sdsl": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/js-sdsl/-/js-sdsl-4.2.0.tgz", - "integrity": "sha512-dyBIzQBDkCqCu+0upx25Y2jGdbTGxE9fshMsCdK0ViOongpV+n5tXRcZY9v7CaVQ79AGS9KA1KHtojxiM7aXSQ==", + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/js-sdsl/-/js-sdsl-4.4.0.tgz", + "integrity": "sha512-FfVSdx6pJ41Oa+CF7RDaFmTnCaFhua+SNYQX74riGOpl96x+2jQCqEfQ2bnXu/5DPCqlRuiqyvTJM0Qjz26IVg==", "dev": true, "funding": { "type": "opencollective", @@ -1424,9 +1436,9 @@ } }, "node_modules/punycode": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", + "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", "dev": true, "engines": { "node": ">=6" @@ -1453,9 +1465,9 @@ ] }, "node_modules/readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", "dev": true, "dependencies": { "core-util-is": "~1.0.0", @@ -1467,40 +1479,16 @@ "util-deprecate": "~1.0.1" } }, - "node_modules/recursive-readdir-synchronous": { - "version": "0.0.4", - "resolved": "https://registry.npmjs.org/recursive-readdir-synchronous/-/recursive-readdir-synchronous-0.0.4.tgz", - "integrity": "sha512-TKglvNd53H9nSeSIX0j6wVpYfO7wc7/Lb9ugL+Qa/yNL17SmAZjZeNdaPtsAsdwp+0iO3zq25NKocEcEec/uRw==", - "dev": true, - "dependencies": { - "minimatch": "3.0.4" - }, - "engines": { - "node": ">=0.6.6" - } - }, - "node_modules/recursive-readdir-synchronous/node_modules/minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "node_modules/recursive-readdir": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/recursive-readdir/-/recursive-readdir-2.2.3.tgz", + "integrity": "sha512-8HrF5ZsXk5FAH9dgsx3BlUer73nIhuj+9OrQwEbLTPOBzGkL1lsFCR01am+v+0m2Cmbs1nP12hLDl5FA7EszKA==", "dev": true, "dependencies": { - "brace-expansion": "^1.1.7" + "minimatch": "^3.0.5" }, "engines": { - "node": "*" - } - }, - "node_modules/regexpp": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.2.0.tgz", - "integrity": "sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg==", - "dev": true, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/mysticatea" + "node": ">=6.0.0" } }, "node_modules/require-dir": { @@ -1576,14 +1564,14 @@ "dev": true }, "node_modules/selenium-webdriver": { - "version": "4.7.1", - "resolved": "https://registry.npmjs.org/selenium-webdriver/-/selenium-webdriver-4.7.1.tgz", - "integrity": "sha512-IfTM9OE8HtCKjOJwyudbAVtAHQKOJK8mu2qrXXbKyj4lqgXF+2lYW4rSZXCV6SLQRWZ+DVGkomCmFzq5orD/ZA==", + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/selenium-webdriver/-/selenium-webdriver-4.9.0.tgz", + "integrity": "sha512-QGaPoREo7sgOVhTiAvCasoi1f4ruTaJDtp0RKNFIbfyns5smK5+iCwnRTIPXb0R3CAYdaqUXd6BHduh37DorzQ==", "dev": true, "dependencies": { - "jszip": "^3.10.0", + "jszip": "^3.10.1", "tmp": "^0.2.1", - "ws": ">=8.7.0" + "ws": ">=8.13.0" }, "engines": { "node": ">= 14.20.0" @@ -1807,16 +1795,16 @@ "dev": true }, "node_modules/ws": { - "version": "8.11.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz", - "integrity": "sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==", + "version": "8.13.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz", + "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==", "dev": true, "engines": { "node": ">=10.0.0" }, "peerDependencies": { "bufferutil": "^4.0.1", - "utf-8-validate": "^5.0.2" + "utf-8-validate": ">=5.0.2" }, "peerDependenciesMeta": { "bufferutil": { @@ -1851,16 +1839,31 @@ } }, "dependencies": { + "@eslint-community/eslint-utils": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", + "integrity": "sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==", + "dev": true, + "requires": { + "eslint-visitor-keys": "^3.3.0" + } + }, + "@eslint-community/regexpp": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.5.0.tgz", + "integrity": "sha512-vITaYzIcNmjn5tF5uxcZ/ft7/RXGrMUIS9HalWckEOF6ESiwXKoMzAQf2UW0aVd6rnOeExTJVd5hmWXucBKGXQ==", + "dev": true + }, "@eslint/eslintrc": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.3.3.tgz", - "integrity": "sha512-uj3pT6Mg+3t39fvLrj8iuCIJ38zKO9FpGtJ4BBJebJhEwjoT+KLVNCcHT5QC9NGRIEi7fZ0ZR8YRb884auB4Lg==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.0.2.tgz", + "integrity": "sha512-3W4f5tDUra+pA+FzgugqL2pRimUTDJWKr7BINqOpkZrC0uYI0NIc0/JFgBROCU07HR6GieA5m3/rsPIhDmCXTQ==", "dev": true, "requires": { "ajv": "^6.12.4", "debug": "^4.3.2", - "espree": "^9.4.0", - "globals": "^13.15.0", + "espree": "^9.5.1", + "globals": "^13.19.0", "ignore": "^5.2.0", "import-fresh": "^3.2.1", "js-yaml": "^4.1.0", @@ -1868,6 +1871,12 @@ "strip-json-comments": "^3.1.1" } }, + "@eslint/js": { + "version": "8.38.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.38.0.tgz", + "integrity": "sha512-IoD2MfUnOV58ghIHCiil01PcohxjbYR/qCxsoC+xNgUwh1EY8jOOrYmu3d3a71+tJJ23uscEV4X2HJWMsPJu4g==", + "dev": true + }, "@humanwhocodes/config-array": { "version": "0.11.8", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.8.tgz", @@ -1924,9 +1933,9 @@ "dev": true }, "@types/node": { - "version": "18.11.15", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.15.tgz", - "integrity": "sha512-VkhBbVo2+2oozlkdHXLrb3zjsRkpdnaU2bXmX8Wgle3PUi569eLRaHGlgETQHR7lLL1w7GiG3h9SnePhxNDecw==", + "version": "18.15.13", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.15.13.tgz", + "integrity": "sha512-N+0kuo9KgrUQ1Sn/ifDXsvg0TTleP7rIy4zOBGECxAljqvqfqpTfzx0Q1NUedOixRMBfe2Whhb056a42cWs26Q==", "dev": true, "optional": true }, @@ -1948,9 +1957,9 @@ "requires": {} }, "acorn": { - "version": "8.8.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.1.tgz", - "integrity": "sha512-7zFpHzhnqYKrkYdUjF1HI1bzd0VygEGX8lFk4k5zVMqHEoES+P+7TKI+EvLO9WVMJ8eekdO0aDEK044xTXwPPA==", + "version": "8.8.2", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.2.tgz", + "integrity": "sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw==", "dev": true }, "acorn-jsx": { @@ -2204,13 +2213,16 @@ "dev": true }, "eslint": { - "version": "8.29.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.29.0.tgz", - "integrity": "sha512-isQ4EEiyUjZFbEKvEGJKKGBwXtvXX+zJbkVKCgTuB9t/+jUBcy8avhkEwWJecI15BkRkOYmvIM5ynbhRjEkoeg==", + "version": "8.38.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.38.0.tgz", + "integrity": "sha512-pIdsD2jwlUGf/U38Jv97t8lq6HpaU/G9NKbYmpWpZGw3LdTNhZLbJePqxOXGB5+JEKfOPU/XLxYxFh03nr1KTg==", "dev": true, "requires": { - "@eslint/eslintrc": "^1.3.3", - "@humanwhocodes/config-array": "^0.11.6", + "@eslint-community/eslint-utils": "^4.2.0", + "@eslint-community/regexpp": "^4.4.0", + "@eslint/eslintrc": "^2.0.2", + "@eslint/js": "8.38.0", + "@humanwhocodes/config-array": "^0.11.8", "@humanwhocodes/module-importer": "^1.0.1", "@nodelib/fs.walk": "^1.2.8", "ajv": "^6.10.0", @@ -2220,16 +2232,15 @@ "doctrine": "^3.0.0", "escape-string-regexp": "^4.0.0", "eslint-scope": "^7.1.1", - "eslint-utils": "^3.0.0", - "eslint-visitor-keys": "^3.3.0", - "espree": "^9.4.0", - "esquery": "^1.4.0", + "eslint-visitor-keys": "^3.4.0", + "espree": "^9.5.1", + "esquery": "^1.4.2", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", "file-entry-cache": "^6.0.1", "find-up": "^5.0.0", "glob-parent": "^6.0.2", - "globals": "^13.15.0", + "globals": "^13.19.0", "grapheme-splitter": "^1.0.4", "ignore": "^5.2.0", "import-fresh": "^3.0.0", @@ -2244,7 +2255,6 @@ "minimatch": "^3.1.2", "natural-compare": "^1.4.0", "optionator": "^0.9.1", - "regexpp": "^3.2.0", "strip-ansi": "^6.0.1", "strip-json-comments": "^3.1.0", "text-table": "^0.2.0" @@ -2255,59 +2265,42 @@ "requires": { "commander": "^2.19.0", "find-root": "^1.1.0", - "recursive-readdir-synchronous": "0.0.4", + "recursive-readdir": "^2.2.3", "require-dir": "^1.2.0", "uuid": "^3.3.2" } }, "eslint-scope": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.1.1.tgz", - "integrity": "sha512-QKQM/UXpIiHcLqJ5AOyIW7XZmzjkzQXYE54n1++wb0u9V/abW3l9uQnxX8Z5Xd18xyKIMTUAyQ0k1e8pz6LUrw==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.0.tgz", + "integrity": "sha512-DYj5deGlHBfMt15J7rdtyKNq/Nqlv5KfU4iodrQ019XESsRnwXH9KAE0y3cwtUHDo2ob7CypAnCqefh6vioWRw==", "dev": true, "requires": { "esrecurse": "^4.3.0", "estraverse": "^5.2.0" } }, - "eslint-utils": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-3.0.0.tgz", - "integrity": "sha512-uuQC43IGctw68pJA1RgbQS8/NP7rch6Cwd4j3ZBtgo4/8Flj4eGE7ZYSZRN3iq5pVUv6GPdW5Z1RFleo84uLDA==", - "dev": true, - "requires": { - "eslint-visitor-keys": "^2.0.0" - }, - "dependencies": { - "eslint-visitor-keys": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz", - "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==", - "dev": true - } - } - }, "eslint-visitor-keys": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.3.0.tgz", - "integrity": "sha512-mQ+suqKJVyeuwGYHAdjMFqjCyfl8+Ldnxuyp3ldiMBFKkvytrXUZWaiPCEav8qDHKty44bD+qV1IP4T+w+xXRA==", + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.0.tgz", + "integrity": "sha512-HPpKPUBQcAsZOsHAFwTtIKcYlCje62XB7SEAcxjtmW6TD1WVpkS6i6/hOVtTZIl4zGj/mBqpFVGvaDneik+VoQ==", "dev": true }, "espree": { - "version": "9.4.1", - "resolved": "https://registry.npmjs.org/espree/-/espree-9.4.1.tgz", - "integrity": "sha512-XwctdmTO6SIvCzd9810yyNzIrOrqNYV9Koizx4C/mRhf9uq0o4yHoCEU/670pOxOL/MSraektvSAji79kX90Vg==", + "version": "9.5.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.5.1.tgz", + "integrity": "sha512-5yxtHSZXRSW5pvv3hAlXM5+/Oswi1AUFqBmbibKb5s6bp3rGIDkyXU6xCoyuuLhijr4SFwPrXRoZjz0AZDN9tg==", "dev": true, "requires": { "acorn": "^8.8.0", "acorn-jsx": "^5.3.2", - "eslint-visitor-keys": "^3.3.0" + "eslint-visitor-keys": "^3.4.0" } }, "esquery": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.0.tgz", - "integrity": "sha512-cCDispWt5vHHtwMY2YrAQ4ibFkAL8RbH5YGBnZBc90MolvvfkkQcJro/aZiAQUlQ3qgrYS6D6v8Gc5G5CQsc9w==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz", + "integrity": "sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==", "dev": true, "requires": { "estraverse": "^5.1.0" @@ -2389,9 +2382,9 @@ "dev": true }, "fastq": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.14.0.tgz", - "integrity": "sha512-eR2D+V9/ExcbF9ls441yIuN6TI2ED1Y2ZcA5BmMtJsOkWOFRJQ0Jt0g1UwqXJJVAb+V+umH5Dfr8oh4EVP7VVg==", + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", + "integrity": "sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==", "dev": true, "requires": { "reusify": "^1.0.4" @@ -2501,9 +2494,9 @@ } }, "globals": { - "version": "13.19.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.19.0.tgz", - "integrity": "sha512-dkQ957uSRWHw7CFXLUtUHQI3g3aWApYhfNR2O6jn/907riyTYKVBmxYVROkBcY614FSSeSJh7Xm7SrUWCxvJMQ==", + "version": "13.20.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.20.0.tgz", + "integrity": "sha512-Qg5QtVkCy/kv3FUSlu4ukeZDVf9ee0iXLAUYX13gbR17bnejFTzr4iS9bY7kwCf1NztRNm1t91fjOiyx4CSwPQ==", "dev": true, "requires": { "type-fest": "^0.20.2" @@ -2524,9 +2517,9 @@ } }, "graceful-fs": { - "version": "4.2.10", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", - "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", "dev": true }, "grapheme-splitter": { @@ -2552,9 +2545,9 @@ } }, "ignore": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.1.tgz", - "integrity": "sha512-d2qQLzTJ9WxQftPAuEQpSPmKqzxePjzVbpAVv62AQ64NTL+wR4JkrVqR/LqFsFEUsHDAiId52mJteHDFuDkElA==", + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz", + "integrity": "sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ==", "dev": true }, "immediate": { @@ -2670,9 +2663,9 @@ "dev": true }, "js-sdsl": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/js-sdsl/-/js-sdsl-4.2.0.tgz", - "integrity": "sha512-dyBIzQBDkCqCu+0upx25Y2jGdbTGxE9fshMsCdK0ViOongpV+n5tXRcZY9v7CaVQ79AGS9KA1KHtojxiM7aXSQ==", + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/js-sdsl/-/js-sdsl-4.4.0.tgz", + "integrity": "sha512-FfVSdx6pJ41Oa+CF7RDaFmTnCaFhua+SNYQX74riGOpl96x+2jQCqEfQ2bnXu/5DPCqlRuiqyvTJM0Qjz26IVg==", "dev": true }, "js-yaml": { @@ -2909,9 +2902,9 @@ } }, "punycode": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", + "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", "dev": true }, "queue-microtask": { @@ -2921,9 +2914,9 @@ "dev": true }, "readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", "dev": true, "requires": { "core-util-is": "~1.0.0", @@ -2935,32 +2928,15 @@ "util-deprecate": "~1.0.1" } }, - "recursive-readdir-synchronous": { - "version": "0.0.4", - "resolved": "https://registry.npmjs.org/recursive-readdir-synchronous/-/recursive-readdir-synchronous-0.0.4.tgz", - "integrity": "sha512-TKglvNd53H9nSeSIX0j6wVpYfO7wc7/Lb9ugL+Qa/yNL17SmAZjZeNdaPtsAsdwp+0iO3zq25NKocEcEec/uRw==", + "recursive-readdir": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/recursive-readdir/-/recursive-readdir-2.2.3.tgz", + "integrity": "sha512-8HrF5ZsXk5FAH9dgsx3BlUer73nIhuj+9OrQwEbLTPOBzGkL1lsFCR01am+v+0m2Cmbs1nP12hLDl5FA7EszKA==", "dev": true, "requires": { - "minimatch": "3.0.4" - }, - "dependencies": { - "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "dev": true, - "requires": { - "brace-expansion": "^1.1.7" - } - } + "minimatch": "^3.0.5" } }, - "regexpp": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.2.0.tgz", - "integrity": "sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg==", - "dev": true - }, "require-dir": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/require-dir/-/require-dir-1.2.0.tgz", @@ -3004,14 +2980,14 @@ "dev": true }, "selenium-webdriver": { - "version": "4.7.1", - "resolved": "https://registry.npmjs.org/selenium-webdriver/-/selenium-webdriver-4.7.1.tgz", - "integrity": "sha512-IfTM9OE8HtCKjOJwyudbAVtAHQKOJK8mu2qrXXbKyj4lqgXF+2lYW4rSZXCV6SLQRWZ+DVGkomCmFzq5orD/ZA==", + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/selenium-webdriver/-/selenium-webdriver-4.9.0.tgz", + "integrity": "sha512-QGaPoREo7sgOVhTiAvCasoi1f4ruTaJDtp0RKNFIbfyns5smK5+iCwnRTIPXb0R3CAYdaqUXd6BHduh37DorzQ==", "dev": true, "requires": { - "jszip": "^3.10.0", + "jszip": "^3.10.1", "tmp": "^0.2.1", - "ws": ">=8.7.0" + "ws": ">=8.13.0" } }, "setimmediate": { @@ -3177,9 +3153,9 @@ "dev": true }, "ws": { - "version": "8.11.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz", - "integrity": "sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==", + "version": "8.13.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz", + "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==", "dev": true, "requires": {} }, diff --git a/package.json b/package.json index 6421bfb527c..7572b75b5b4 100644 --- a/package.json +++ b/package.json @@ -15,9 +15,8 @@ }, "devDependencies": { "@zotero/eslint-config": "^1.0.5", - "chalk": "^4.1.0", "chromedriver": "^95.0.0", - "eslint": "^8.29.0", + "eslint": "^8.38.0", "eslint-plugin-zotero-translator": "file:.ci/eslint-plugin-zotero-translator", "selenium-webdriver": "^4.0.0-alpha.7" } From 8c803339b6b0a1b833716a85769da17afe3fc3f1 Mon Sep 17 00:00:00 2001 From: Sebastian Karcher Date: Mon, 24 Apr 2023 21:24:18 -0400 Subject: [PATCH 021/125] EM: Detect chaper for inbook_title (#3028) --- Embedded Metadata.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Embedded Metadata.js b/Embedded Metadata.js index 24d748d3121..5b7f264aac9 100644 --- a/Embedded Metadata.js +++ b/Embedded Metadata.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2023-01-23 17:12:53" + "lastUpdated": "2023-04-24 14:42:56" } /* @@ -316,6 +316,7 @@ function init(doc, url, callback, forceLoadRDF) { hwType = "conferencePaper"; break; case "citation_book_title": + case "citation_inbook_title": hwType = "bookSection"; break; case "citation_dissertation_institution": @@ -735,7 +736,7 @@ function tryOgAuthors(doc) { var authors = []; var ogAuthors = ZU.xpath(doc, '//meta[@property="article:author" or @property="video:director" or @property="music:musician"]'); for (var i = 0; i < ogAuthors.length; i++) { - if (ogAuthors[i].content && ogAuthors[i].content.search(/(https?:\/\/)?[\da-z.-]+\.[a-z.]{2,6}/) < 0 && ogAuthors[i].content !== "false") { + if (ogAuthors[i].content && /(https?:\/\/)?[\da-z.-]+\.[a-z.]{2,6}/.test(ogAuthors[i].content) && ogAuthors[i].content !== "false") { authors.push(ZU.cleanAuthor(ogAuthors[i].content, "author")); } } From ecb80fad3bcd83e07c27420e9d86ef2286ff2c96 Mon Sep 17 00:00:00 2001 From: Dan Stillman Date: Thu, 27 Apr 2023 05:24:31 -0400 Subject: [PATCH 022/125] Lulu: Switch to HTTPS and disable ISBN search Translator probably has to be rewritten if we still want it, but for now don't use it for ISBN lookup. --- Lulu.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Lulu.js b/Lulu.js index 3b9fe072be8..cb9752c3765 100644 --- a/Lulu.js +++ b/Lulu.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 12, "browserSupport": "gcsibv", - "lastUpdated": "2016-11-04 21:18:44" + "lastUpdated": "2023-04-27 09:24:18" } function getSearchResults(doc) { @@ -101,6 +101,9 @@ function makeItem(doc, url) { } function detectSearch(items) { + // Disabled -- no longer working + return false; + if (items.ISBN) return true; if (!items.length) return; @@ -120,7 +123,7 @@ function doSearch(items) { var isbn; if (items[i].ISBN && (isbn = ZU.cleanISBN('' + items[i].ISBN))) { (function(item, isbn) { - ZU.processDocuments('http://www.lulu.com/shop/search.ep?keyWords=' + isbn, function(doc, url) { + ZU.processDocuments('https://www.lulu.com/shop/search.ep?keyWords=' + isbn, function(doc, url) { var results = getSearchResults(doc); if (!results.length) { if (item.complete) item.complete(); @@ -258,4 +261,4 @@ var testCases = [ ] } ] -/** END TEST CASES **/ \ No newline at end of file +/** END TEST CASES **/ From 402c668fb35d5207a204a0f0292c137cb8ba476b Mon Sep 17 00:00:00 2001 From: Sebastian Karcher Date: Fri, 28 Apr 2023 07:07:46 -0400 Subject: [PATCH 023/125] Don't detect J-Stage PDFs (#3030) --- J-Stage.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/J-Stage.js b/J-Stage.js index c8dfa00f3a2..0371b96cc84 100644 --- a/J-Stage.js +++ b/J-Stage.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2019-06-15 19:04:11" + "lastUpdated": "2023-04-27 13:09:03" } /* @@ -35,7 +35,8 @@ */ function detectWeb(doc, url) { - if (url.includes("/article/")) { + // don't detect on PDF pages + if (url.includes("/article/") && !url.includes("/_pdf")) { return "journalArticle"; } else if ((url.includes("/result/") || url.includes("/browse/")) From 8a7b9fac0ba35eeb6e28b7b50160712eaf186c9f Mon Sep 17 00:00:00 2001 From: Brendan O'Connell <68507084+brendan-oconnell@users.noreply.github.com> Date: Fri, 5 May 2023 03:14:38 +0200 Subject: [PATCH 024/125] Fixed selector for getting abstract from page (#3033) --- SAGE Journals.js | 91 ++++++++++++++++++++++++++++++------------------ 1 file changed, 57 insertions(+), 34 deletions(-) diff --git a/SAGE Journals.js b/SAGE Journals.js index 8999cadc142..c48c26dacd3 100644 --- a/SAGE Journals.js +++ b/SAGE Journals.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2020-12-06 18:57:06" + "lastUpdated": "2023-05-04 14:14:25" } /* @@ -107,7 +107,7 @@ function scrape(doc, url) { if (text.includes("DA - ")) { text = text.replace(/Y1[ ]{2}- .*\r?\n/, ''); } - + var translator = Zotero.loadTranslator("import"); translator.setTranslator("32d59d2d-b65a-4da4-b0a3-bdd3cfb979e7"); translator.setString(text); @@ -125,12 +125,11 @@ function scrape(doc, url) { } // The encoding of apostrophs in the RIS are incorrect and // therefore we extract the abstract again from the website. - var abstract = ZU.xpathText(doc, '//article//div[contains(@class, "abstractSection")]/p'); + var abstract = doc.querySelector("#abstract > div").innerText; if (abstract) { item.abstractNote = abstract; } - - + for (let tag of tags) { item.tags.push(tag.textContent); } @@ -142,7 +141,7 @@ function scrape(doc, url) { item.creators[i] = ZU.cleanAuthor(item.creators[i].lastName, type, comma); } } - + item.notes = []; item.language = ZU.xpathText(doc, '//meta[@name="dc.Language"]/@content'); item.attachments.push({ @@ -160,24 +159,24 @@ function scrape(doc, url) { var testCases = [ { "type": "web", - "url": "http://journals.sagepub.com/doi/abs/10.1177/1754073910380971", + "url": "https://journals.sagepub.com/doi/abs/10.1177/1754073910380971", + "detectedItemType": "journalArticle", "items": [ { "itemType": "journalArticle", "title": "Emotion and Regulation are One!", "creators": [ { - "firstName": "Arvid", "lastName": "Kappas", + "firstName": "Arvid", "creatorType": "author" } ], - "date": "January 1, 2011", + "date": "2011-01-01", "DOI": "10.1177/1754073910380971", "ISSN": "1754-0739", "abstractNote": "Emotions are foremost self-regulating processes that permit rapid responses and adaptations to situations of personal concern. They have biological bases and are shaped ontogenetically via learning and experience. Many situations and events of personal concern are social in nature. Thus, social exchanges play an important role in learning about rules and norms that shape regulation processes. I argue that (a) emotions often are actively auto-regulating—the behavior implied by the emotional reaction bias to the eliciting event or situation modifies or terminates the situation; (b) certain emotion components are likely to habituate dynamically, modifying the emotional states; (c) emotions are typically intra- and interpersonal processes at the same time, and modulating forces at these different levels interact; (d) emotions are not just regulated—they regulate. Important conclusions of my arguments are that the scientific analysis of emotion should not exclude regulatory processes, and that effortful emotion regulation should be seen relative to a backdrop of auto-regulation and habituation, and not the ideal notion of a neutral baseline. For all practical purposes unregulated emotion is not a realistic concept.", "issue": "1", - "journalAbbreviation": "Emotion Review", "language": "en", "libraryCatalog": "SAGE Journals", "pages": "17-25", @@ -190,17 +189,7 @@ var testCases = [ "mimeType": "application/pdf" } ], - "tags": [ - { - "tag": "emotion regulation" - }, - { - "tag": "facial expression" - }, - { - "tag": "facial feedback" - } - ], + "tags": [], "notes": [], "seeAlso": [] } @@ -208,8 +197,9 @@ var testCases = [ }, { "type": "web", - "url": "http://journals.sagepub.com/toc/rera/86/3", - "items": "multiple" + "url": "https://journals.sagepub.com/toc/rera/86/3", + "detectedItemType": false, + "items": [] }, { "type": "web", @@ -359,6 +349,7 @@ var testCases = [ { "type": "web", "url": "https://journals.sagepub.com/doi/10.1177/0263276404046059", + "detectedItemType": "journalArticle", "items": [ { "itemType": "journalArticle", @@ -370,12 +361,11 @@ var testCases = [ "creatorType": "author" } ], - "date": "October 1, 2004", + "date": "2004-10-01", "DOI": "10.1177/0263276404046059", "ISSN": "0263-2764", "abstractNote": "This article is concerned with how to conceptualize and theorize the nature of the ‘car system’ that is a particularly key, if surprisingly neglected, element in ‘globalization’. The article deploys the notion of systems as self-reproducing or autopoietic. This notion is used to understand the origins of the 20th-century car system and especially how its awesome pattern of path dependency was established and exerted a particularly powerful and self-expanding pattern of domination across the globe. The article further considers whether and how the 20th-century car system may be transcended. It elaborates a number of small changes that are now occurring in various test sites, factories, ITC sites, cities and societies. The article briefly considers whether these small changes may in their contingent ordering end this current car system. The article assesses whether such a new system could emerge well before the end of this century, whether in other words some small changes now may produce the very large effect of a new post-car system that would have great implications for urban life, for mobility and for limiting projected climate change.", "issue": "4-5", - "journalAbbreviation": "Theory, Culture & Society", "language": "en", "libraryCatalog": "SAGE Journals", "pages": "25-39", @@ -388,23 +378,56 @@ var testCases = [ "mimeType": "application/pdf" } ], - "tags": [ - { - "tag": "automobility" - }, + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://journals.sagepub.com/doi/10.1177/1071181322661302", + "detectedItemType": "journalArticle", + "items": [ + { + "itemType": "journalArticle", + "title": "Deidentification of Drivers’ Face Videos: Scope and Challenges in Human Factors Research", + "creators": [ { - "tag": "path dependence" + "lastName": "Thapa", + "firstName": "Surendrabikram", + "creatorType": "author" }, { - "tag": "technology" + "lastName": "Cook", + "firstName": "Julie", + "creatorType": "author" }, { - "tag": "time-space" - }, + "lastName": "Sarkar", + "firstName": "Abhijit", + "creatorType": "author" + } + ], + "date": "2022-09-01", + "DOI": "10.1177/1071181322661302", + "ISSN": "2169-5067", + "abstractNote": "Data sharing across disciplines helps to build collaboration, and advance research. With recent development in data-driven models, there is an unprecedented need for data. However, data collected from human research subjects are required to follow proper ethical guidelines. Researchers have an obligation to protect the privacy of research participants and address ethical and safety concerns when data contains personally identifying information (PII). This paper addresses this problem with a focus on sharing drivers’ face videos for transportation research. The paper first gives an overview of the multitude of problems that are associated with sharing drivers’ videos. Then it demonstrates the possible directions for data sharing by de-identifying drivers’ faces using artificial intelligence-based techniques. The results achieved through the proposed techniques were evaluated qualitatively and quantitatively to prove the validity of the suggested methods. We specifically demonstrated how face-swapping algorithms can effectively de-identify faces while still preserving important attributes related to human factor research including eye movements, head movements, mouth movements, etc. Finally, we discuss possible measures to share such de-identified videos with the greater research community.", + "issue": "1", + "language": "en", + "libraryCatalog": "SAGE Journals", + "pages": "1509-1513", + "publicationTitle": "Proceedings of the Human Factors and Ergonomics Society Annual Meeting", + "shortTitle": "Deidentification of Drivers’ Face Videos", + "url": "https://doi.org/10.1177/1071181322661302", + "volume": "66", + "attachments": [ { - "tag": "tipping point" + "title": "SAGE PDF Full Text", + "mimeType": "application/pdf" } ], + "tags": [], "notes": [], "seeAlso": [] } From 6300365a9ea34fae81d3c93ceebd075cb0973fb1 Mon Sep 17 00:00:00 2001 From: Sebastian Karcher Date: Mon, 8 May 2023 07:46:46 -0400 Subject: [PATCH 025/125] Add Dataset to RIS (#3022) --- RIS.js | 1211 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 701 insertions(+), 510 deletions(-) diff --git a/RIS.js b/RIS.js index b74a3cb989b..9fd7761af8f 100644 --- a/RIS.js +++ b/RIS.js @@ -17,9 +17,33 @@ }, "inRepository": true, "translatorType": 3, - "lastUpdated": "2022-07-14 15:44:17" + "lastUpdated": "2023-04-20 19:03:44" } +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2006-2023 Simon Kornblith, Aurimas Vinckevicus, Abe Jellinek + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + + function detectImport() { var line; var i = 0; @@ -28,20 +52,20 @@ function detectImport() { if (line != "") { if (line.substr(0, 6).match(/^TY {1,2}- /)) { return true; - } else { - if (i++ > 3) { - return false; - } + } + else if (i++ > 3) { + return false; } } } + return false; } /******************** * Exported options * ********************/ - //exported as translatorObject.options - var exportedOptions = { +//exported as translatorObject.options +var exportedOptions = { itemType: false, //allows translators to override item type defaultItemType: false, //item type to default to typeMap: false, @@ -57,33 +81,34 @@ var DEFAULT_EXPORT_TYPE = 'GEN'; var DEFAULT_IMPORT_TYPE = 'journalArticle'; var exportTypeMap = { - artwork:"ART", - audioRecording:"SOUND", //consider MUSIC - bill:"BILL", - blogPost:"BLOG", - book:"BOOK", - bookSection:"CHAP", - "case":"CASE", - computerProgram:"COMP", - conferencePaper:"CONF", - dictionaryEntry:"DICT", - encyclopediaArticle:"ENCYC", - email:"ICOMM", - film:"MPCT", - hearing:"HEAR", - journalArticle:"JOUR", - letter:"PCOMM", - magazineArticle:"MGZN", - manuscript:"MANSCPT", - map:"MAP", - newspaperArticle:"NEWS", - patent:"PAT", - presentation:"SLIDE", - report:"RPRT", - statute:"STAT", - thesis:"THES", - videoRecording:"VIDEO", - webpage:"ELEC" + artwork: "ART", + audioRecording: "SOUND", //consider MUSIC + bill: "BILL", + blogPost: "BLOG", + book: "BOOK", + bookSection: "CHAP", + case: "CASE", + computerProgram: "COMP", + conferencePaper: "CONF", + dictionaryEntry: "DICT", + encyclopediaArticle: "ENCYC", + email: "ICOMM", + dataset: "DATA", + film: "MPCT", + hearing: "HEAR", + journalArticle: "JOUR", + letter: "PCOMM", + magazineArticle: "MGZN", + manuscript: "MANSCPT", + map: "MAP", + newspaperArticle: "NEWS", + patent: "PAT", + presentation: "SLIDE", + report: "RPRT", + statute: "STAT", + thesis: "THES", + videoRecording: "VIDEO", + webpage: "ELEC" }; //These export type maps are degenerate @@ -91,49 +116,48 @@ var exportTypeMap = { //These should either be duplicates of some of the RIS types above // or be different from the importTypeMap mappings var degenerateExportTypeMap = { - interview:"PCOMM", - instantMessage:"ICOMM", - forumPost:"ICOMM", - tvBroadcast:"MPCT", - radioBroadcast:"SOUND", - podcast:"SOUND", - document:"GEN" //imported as journalArticle + interview: "PCOMM", + instantMessage: "ICOMM", + forumPost: "ICOMM", + tvBroadcast: "MPCT", + radioBroadcast: "SOUND", + podcast: "SOUND", + document: "GEN" //imported as journalArticle }; //These are degenerate types that are not exported as the same TY value //These should not include any types from exportTypeMap //We add the rest from exportTypeMap var importTypeMap = { - ABST:"journalArticle", - ADVS:"film", - AGGR:"document", //how can we handle "database" citations? - ANCIENT:"document", - CHART:"artwork", - CLSWK:"book", - CPAPER:"conferencePaper", - CTLG:"magazineArticle", - DATA:"document", //dataset - DBASE:"document", //database - EBOOK:"book", - ECHAP:"bookSection", - EDBOOK:"book", - EJOUR:"journalArticle", - EQUA:"document", //what's a good way to handle this? - FIGURE:"artwork", - GEN:"journalArticle", - GOVDOC:"report", - GRNT:"document", - INPR:"manuscript", - JFULL:"journalArticle", - LEGAL:"case", //is this what they mean? - MULTI:"videoRecording", //maybe? - MUSIC:"audioRecording", - PAMP:"manuscript", - SER:"book", - STAND:"report", - UNBILL:"manuscript", - UNPD:"manuscript", - WEB:"webpage" //not in spec, but used by EndNote + ABST: "journalArticle", + ADVS: "film", + AGGR: "document", //how can we handle "database" citations? + ANCIENT: "document", + CHART: "artwork", + CLSWK: "book", + CPAPER: "conferencePaper", + CTLG: "magazineArticle", + DBASE: "dataset", //database + EBOOK: "book", + ECHAP: "bookSection", + EDBOOK: "book", + EJOUR: "journalArticle", + EQUA: "document", //what's a good way to handle this? + FIGURE: "artwork", + GEN: "journalArticle", + GOVDOC: "report", + GRNT: "document", + INPR: "manuscript", + JFULL: "journalArticle", + LEGAL: "case", //is this what they mean? + MULTI: "videoRecording", //maybe? + MUSIC: "audioRecording", + PAMP: "manuscript", + SER: "book", + STAND: "report", + UNBILL: "manuscript", + UNPD: "manuscript", + WEB: "webpage" //not in spec, but used by EndNote }; //supplement input map with export @@ -142,6 +166,14 @@ for (ty in exportTypeMap) { importTypeMap[exportTypeMap[ty]] = ty; } +// for Zotero clients <6.0.24: set dataset back to document +if (!ZU.fieldIsValidForType('title', 'dataset')) { + delete importTypeMap.DATA; + delete importTypeMap.DBASE; + importTypeMap.DBASE = 'document'; + importTypeMap.DATA = 'document'; +} + //merge degenerate export type map into main list for (ty in degenerateExportTypeMap) { exportTypeMap[ty] = degenerateExportTypeMap[ty]; @@ -174,227 +206,231 @@ for (ty in degenerateExportTypeMap) { //(except for item types that do not have unique RIS types, see above) var fieldMap = { //same for all itemTypes - AB:"abstractNote", - AN:"archiveLocation", - CN:"callNumber", - DB:"archive", - DO:"DOI", - DP:"libraryCatalog", - J2:"journalAbbreviation", - KW:"tags", - L1:"attachments/PDF", - L2:"attachments/HTML", - L4:"attachments/other", - N1:"notes", - ST:"shortTitle", - UR:"url", - Y2:"accessDate", + AB: "abstractNote", + AN: "archiveLocation", + CN: "callNumber", + DB: "archive", + DO: "DOI", + DP: "libraryCatalog", + J2: "journalAbbreviation", + KW: "tags", + L1: "attachments/PDF", + L2: "attachments/HTML", + L4: "attachments/other", + N1: "notes", + ST: "shortTitle", + UR: "url", + Y2: "accessDate", //type specific //tag => field:itemTypes //if itemType not explicitly given, __default field is used // unless itemType is excluded in __exclude TI: { - "__default":"title", - subject:["email"], - caseName:["case"], - nameOfAct:["statute"] + __default: "title", + subject: ["email"], + caseName: ["case"], + nameOfAct: ["statute"] }, T2: { - code:["bill", "statute"], - bookTitle:["bookSection"], - blogTitle:["blogPost"], - conferenceName:["conferencePaper"], - dictionaryTitle:["dictionaryEntry"], - encyclopediaTitle:["encyclopediaArticle"], - committee:["hearing"], - forumTitle:["forumPost"], - websiteTitle:["webpage"], - programTitle:["radioBroadcast", "tvBroadcast"], - meetingName:["presentation"], - seriesTitle:["computerProgram", "map", "report"], + code: ["bill", "statute"], + bookTitle: ["bookSection"], + blogTitle: ["blogPost"], + conferenceName: ["conferencePaper"], + dictionaryTitle: ["dictionaryEntry"], + encyclopediaTitle: ["encyclopediaArticle"], + committee: ["hearing"], + forumTitle: ["forumPost"], + websiteTitle: ["webpage"], + programTitle: ["radioBroadcast", "tvBroadcast"], + meetingName: ["presentation"], + seriesTitle: ["computerProgram", "map", "report"], series: ["book"], - publicationTitle:["journalArticle", "magazineArticle", "newspaperArticle"] + publicationTitle: ["journalArticle", "magazineArticle", "newspaperArticle"] }, T3: { - legislativeBody:["hearing", "bill"], - series:["bookSection", "conferencePaper", "journalArticle"], - seriesTitle:["audioRecording"] + legislativeBody: ["hearing", "bill"], + series: ["bookSection", "conferencePaper", "journalArticle"], + seriesTitle: ["audioRecording"] }, //NOT HANDLED: reviewedAuthor, scriptwriter, contributor, guest AU: { - "__default":"creators/author", - "creators/artist":["artwork"], - "creators/cartographer":["map"], - "creators/composer":["audioRecording"], - "creators/director":["film", "radioBroadcast", "tvBroadcast", "videoRecording"], //this clashes with audioRecording - "creators/interviewee":["interview"], - "creators/inventor":["patent"], - "creators/podcaster":["podcast"], - "creators/programmer":["computerProgram"] + __default: "creators/author", + "creators/artist": ["artwork"], + "creators/cartographer": ["map"], + "creators/composer": ["audioRecording"], + "creators/director": ["film", "radioBroadcast", "tvBroadcast", "videoRecording"], //this clashes with audioRecording + "creators/interviewee": ["interview"], + "creators/inventor": ["patent"], + "creators/podcaster": ["podcast"], + "creators/programmer": ["computerProgram"] }, A1: { - "__default":"creators/author", - "creators/artist":["artwork"], - "creators/cartographer":["map"], - "creators/composer":["audioRecording"], - "creators/director":["film", "radioBroadcast", "tvBroadcast", "videoRecording"], //this clashes with audioRecording - "creators/interviewee":["interview"], - "creators/inventor":["patent"], - "creators/podcaster":["podcast"], - "creators/programmer":["computerProgram"] + __default: "creators/author", + "creators/artist": ["artwork"], + "creators/cartographer": ["map"], + "creators/composer": ["audioRecording"], + "creators/director": ["film", "radioBroadcast", "tvBroadcast", "videoRecording"], //this clashes with audioRecording + "creators/interviewee": ["interview"], + "creators/inventor": ["patent"], + "creators/podcaster": ["podcast"], + "creators/programmer": ["computerProgram"] }, A2: { - "creators/sponsor":["bill"], - "creators/performer":["audioRecording"], - "creators/presenter":["presentation"], - "creators/interviewer":["interview"], - "creators/editor":["journalArticle", "bookSection", "conferencePaper", "dictionaryEntry", "document", "encyclopediaArticle"], - "creators/seriesEditor":["book", "report"], - "creators/recipient":["email", "instantMessage", "letter"], - reporter:["case"], - issuingAuthority:["patent"] + "creators/sponsor": ["bill"], + "creators/performer": ["audioRecording"], + "creators/presenter": ["presentation"], + "creators/interviewer": ["interview"], + "creators/editor": ["journalArticle", "bookSection", "conferencePaper", "dictionaryEntry", "document", "encyclopediaArticle"], + "creators/seriesEditor": ["book", "report"], + "creators/recipient": ["email", "instantMessage", "letter"], + reporter: ["case"], + issuingAuthority: ["patent"] }, - A3: { - "creators/contributor":["thesis"], - "creators/cosponsor":["bill"], - "creators/producer":["film", "tvBroadcast", "videoRecording", "radioBroadcast"], - "creators/editor":["book"], - "creators/seriesEditor":["bookSection", "conferencePaper", "dictionaryEntry", "encyclopediaArticle", "map"] + A3: { + "creators/contributor": ["thesis"], + "creators/cosponsor": ["bill"], + "creators/producer": ["film", "tvBroadcast", "videoRecording", "radioBroadcast"], + "creators/editor": ["book"], + "creators/seriesEditor": ["bookSection", "conferencePaper", "dictionaryEntry", "encyclopediaArticle", "map"] }, A4: { - "__default":"creators/translator", - "creators/counsel":["case"], - "creators/contributor":["conferencePaper", "film"] //translator does not fit these + __default: "creators/translator", + "creators/counsel": ["case"], + "creators/contributor": ["conferencePaper", "film", "dataset"] //translator does not fit these }, C1: { - filingDate:["patent"], //not in spec - "creators/castMember":["radioBroadcast", "tvBroadcast", "videoRecording"], - scale:["map"], - place:["conferencePaper"] + filingDate: ["patent"], //not in spec + "creators/castMember": ["radioBroadcast", "tvBroadcast", "videoRecording"], + scale: ["map"], + place: ["conferencePaper"] }, C2: { - issueDate:["patent"], //not in spec - "creators/bookAuthor":["bookSection"], - "creators/commenter":["blogPost"] + issueDate: ["patent"], //not in spec + "creators/bookAuthor": ["bookSection"], + "creators/commenter": ["blogPost"] }, C3: { - artworkSize:["artwork"], - proceedingsTitle:["conferencePaper"], - country:["patent"] + artworkSize: ["artwork"], + proceedingsTitle: ["conferencePaper"], + country: ["patent"] }, C4: { - "creators/wordsBy":["audioRecording"], //not in spec - "creators/attorneyAgent":["patent"], - genre:["film"] + "creators/wordsBy": ["audioRecording"], //not in spec + "creators/attorneyAgent": ["patent"], + genre: ["film"] }, C5: { - references:["patent"], - audioRecordingFormat:["audioRecording", "radioBroadcast"], - videoRecordingFormat:["film", "tvBroadcast", "videoRecording"] + references: ["patent"], + audioRecordingFormat: ["audioRecording", "radioBroadcast"], + videoRecordingFormat: ["film", "tvBroadcast", "videoRecording"] }, C6: { - legalStatus:["patent"], + legalStatus: ["patent"], }, CY: { - "__default":"place", - "__exclude":["conferencePaper"] //should be exported as C1 + __default: "place", + repositoryLocation: ["dataset"], + __exclude: ["conferencePaper"] //should be exported as C1 }, DA: { //also see PY when editing - "__default":"date", - dateEnacted:["statute"], - dateDecided:["case"], - issueDate:["patent"] + __default: "date", + dateEnacted: ["statute"], + dateDecided: ["case"], + issueDate: ["patent"] }, ET: { - "__default":"edition", -// "__ignore":["journalArticle"], //EPubDate - session:["bill", "hearing", "statute"], - version:["computerProgram"] + __default: "edition", + // "__ignore":["journalArticle"], //EPubDate + session: ["bill", "hearing", "statute"], + versionNumber: ["computerProgram", "dataset"] }, IS: { - "__default":"issue", - numberOfVolumes: ["bookSection"] + __default: "issue", + numberOfVolumes: ["bookSection"], + __exclude: ["dataset"] //IS }, LA: { - "__default":"language", + __default: "language", programmingLanguage: ["computerProgram"] }, M1: { - seriesNumber:["book"], - billNumber:["bill"], - system:["computerProgram"], - documentNumber:["hearing"], - applicationNumber:["patent"], - publicLawNumber:["statute"], - episodeNumber:["podcast", "radioBroadcast", "tvBroadcast"] + seriesNumber: ["book"], + billNumber: ["bill"], + system: ["computerProgram"], + documentNumber: ["hearing"], + applicationNumber: ["patent"], + publicLawNumber: ["statute"], + episodeNumber: ["podcast", "radioBroadcast", "tvBroadcast"] }, M3: { - manuscriptType:["manuscript"], - mapType:["map"], - reportType:["report"], - thesisType:["thesis"], - websiteType:["blogPost", "webpage"], - postType:["forumPost"], - letterType:["letter"], - interviewMedium:["interview"], - presentationType:["presentation"], - artworkMedium:["artwork"], - audioFileType:["podcast"] + manuscriptType: ["manuscript"], + mapType: ["map"], + reportType: ["report"], + thesisType: ["thesis"], + websiteType: ["blogPost", "webpage"], + postType: ["forumPost"], + letterType: ["letter"], + interviewMedium: ["interview"], + presentationType: ["presentation"], + artworkMedium: ["artwork"], + audioFileType: ["podcast"] }, NV: { - "__default": "numberOfVolumes", - "__exclude": ["bookSection"] //IS + __default: "numberOfVolumes", + identifier: ["dataset"], + __exclude: ["bookSection"] //IS }, OP: { - history:["hearing", "statute", "bill", "case"], - priorityNumbers:["patent"] + history: ["hearing", "statute", "bill", "case"], + priorityNumbers: ["patent"] }, PB: { - "__default":"publisher", - label:["audioRecording"], - court:["case"], - distributor:["film"], - assignee:["patent"], - institution:["report"], - university:["thesis"], - company:["computerProgram"], - studio:["videoRecording"], - network:["radioBroadcast", "tvBroadcast"] + __default: "publisher", + label: ["audioRecording"], + court: ["case"], + distributor: ["film"], + assignee: ["patent"], + institution: ["report"], + repository: ["dataset"], + university: ["thesis"], + company: ["computerProgram"], + studio: ["videoRecording"], + network: ["radioBroadcast", "tvBroadcast"] }, PY: { //duplicate of DA, but this will only output year - "__default":"date", - dateEnacted:["statute"], - dateDecided:["case"], - issueDate:["patent"] + __default: "date", + dateEnacted: ["statute"], + dateDecided: ["case"], + issueDate: ["patent"] }, SE: { - "__default": "section", //though this can refer to pages, start page, etc. for some types. Zotero does not support any of those combinations, however. - "__exclude": ["case"] + __default: "section", //though this can refer to pages, start page, etc. for some types. Zotero does not support any of those combinations, however. + __exclude: ["case"] }, SN: { - "__default":"ISBN", - ISSN:["journalArticle", "magazineArticle", "newspaperArticle"], - patentNumber:["patent"], - reportNumber:["report"], + __default: "ISBN", + ISSN: ["journalArticle", "magazineArticle", "newspaperArticle"], + patentNumber: ["patent"], + reportNumber: ["report"], }, SP: { - "__default":"pages", //needs extra processing - codePages:["bill"], //bill - numPages:["book", "thesis", "manuscript"], //manuscript not really in spec - firstPage:["case"], - runningTime:["film"] + __default: "pages", //needs extra processing + codePages: ["bill"], //bill + numPages: ["book", "thesis", "manuscript"], //manuscript not really in spec + firstPage: ["case"], + runningTime: ["film"] }, SV: { seriesNumber: ["bookSection"], docketNumber: ["case"] //not in spec. EndNote exports this way }, VL: { - "__default":"volume", - codeNumber:["statute"], - codeVolume:["bill"], - reporterVolume:["case"], - "__exclude":["patent", "webpage"] + __default: "volume", + codeNumber: ["statute"], + codeVolume: ["bill"], + reporterVolume: ["case"], + __exclude: ["patent", "webpage"] } }; @@ -403,14 +439,14 @@ var fieldMap = { //these are not exported the same way var degenerateImportFieldMap = { AD: { - "__default": "unsupported/Author Address", + __default: "unsupported/Author Address", "unsupported/Inventor Address": ["patent"] }, AV: "archiveLocation", //REFMAN BT: { title: ["book", "manuscript"], bookTitle: ["bookSection"], - "__default": "backupPublicationTitle" //we do more filtering on this later + __default: "backupPublicationTitle" //we do more filtering on this later }, CA: "unsupported/Caption", CR: "rights", @@ -421,15 +457,18 @@ var degenerateImportFieldMap = { H1: "unsupported/Library Catalog", //Citavi specific (possibly multiple occurences) H2: "unsupported/Call Number", //Citavi specific (possibly multiple occurences) ID: "__ignore", + IS: { + versionNumber: ["dataset"] + }, JA: "journalAbbreviation", JF: "publicationTitle", JO: { - "__default": "journalAbbreviation", + __default: "journalAbbreviation", conferenceName: ["conferencePaper"] }, LB: "unsupported/Label", M1: { - "__default":"extra", + __default: "extra", issue: ["journalArticle"], //EndNote hack numberOfVolumes: ["bookSection"], //EndNote exports here instead of IS accessDate: ["webpage"] //this is access date when coming from EndNote @@ -439,18 +478,18 @@ var degenerateImportFieldMap = { N2: "abstractNote", NV: "numberOfVolumes", OP: { - "__default": "unsupported/Original Publication", + __default: "unsupported/Original Publication", "unsupported/Content": ["blogPost", "computerProgram", "film", "presentation", "report", "videoRecording", "webpage"] }, RI: { - "__default":"unsupported/Reviewed Item", + __default: "unsupported/Reviewed Item", "unsupported/Article Number": ["statute"] }, RN: "notes", SE: { "unsupported/File Date": ["case"] }, - T1: fieldMap["TI"], + T1: fieldMap.TI, T2: "backupPublicationTitle", //most item types should be covered above T3: { series: ["book"] @@ -458,15 +497,15 @@ var degenerateImportFieldMap = { TA: "unsupported/Translated Author", TT: "unsupported/Translated Title", VL: { - "unsupported/Patent Version Number":['patent'], + "unsupported/Patent Version Number": ['patent'], accessDate: ["webpage"] //technically access year according to EndNote }, - Y1: fieldMap["DA"] // Old RIS spec + Y1: fieldMap.DA // Old RIS spec }; /** * @class Generic tag mapping with caching - * + * * @param {Tag <-> zotero field map []} mapList An array of field map lists as * described above. Lists are matched in order they are supplied. If a tag is * not present in the list, the next list is checked. If the RIS tag is @@ -477,7 +516,7 @@ var degenerateImportFieldMap = { * the entries in the `mapList`, containing deprecated tags that should only * be used if no other tag maps to the same Zotero item field. */ -var TagMapper = function(mapList, deprecatedMap) { +var TagMapper = function (mapList, deprecatedMap) { this.cache = {}; this.reverseCache = {}; this.mapList = mapList; @@ -486,7 +525,7 @@ var TagMapper = function(mapList, deprecatedMap) { TagMapper.prototype.isDeprecated = function (tag) { return this.deprecatedMap.hasOwnProperty(tag); -} +}; /** * Given an item type and a RIS tag, return Zotero field data should be mapped to. @@ -496,7 +535,7 @@ TagMapper.prototype.isDeprecated = function (tag) { * @param {String} tag RIS tag * @return {String} Zotero field */ -TagMapper.prototype.getField = function(itemType, tag) { +TagMapper.prototype.getField = function (itemType, tag) { if (!this.cache[itemType]) this.cache[itemType] = {}; //retrieve from cache if available @@ -506,9 +545,9 @@ TagMapper.prototype.getField = function(itemType, tag) { } var field = false; - for (var i=0, n=this.mapList.length; i 85 || (lastLineLength !== undefined && lastLineLength < 65) || cleanLine.length == 0) - ) { - + ) { cleanLine = "\n" + cleanLine; newLineAdded = true; } //don't remove new lines from keywords or attachments - if (!newLineAdded && preserveNewLines.indexOf(tagValue.tag) != -1) { + if (!newLineAdded && preserveNewLines.includes(tagValue.tag)) { cleanLine = "\n" + cleanLine; newLineAdded = true; } //check if we need to add a space before concatenating - if (!newLineAdded && tagValue.value.charAt(tagValue.value.length-1) != ' ') { + if (!newLineAdded && tagValue.value.charAt(tagValue.value.length - 1) != ' ') { cleanLine = ' ' + cleanLine; } @@ -743,9 +788,11 @@ var RISReader = new function() { } if (tagValue) return tagValue; + return false; } var _lineBuffer = []; + /** * private * Gets the next line in the buffer or file @@ -757,7 +804,7 @@ var RISReader = new function() { // because we may have an empty line, which could be meaningful if (_lineBuffer.length) return _lineBuffer.pop(); var line = Zotero.read(); - if (line && (line.indexOf('\u2028') != -1 || line.indexOf('\u2029') != -1)) { + if (line && (line.includes('\u2028') || line.includes('\u2029'))) { // Apparently some services think that it's cool to break up single // lines in RIS into shorter lines using Unicode "LINE SEPARATOR" // character. Well, that sucks for us, because . (dot) in regexp does @@ -776,6 +823,7 @@ var RISReader = new function() { * Generic methods for cleaning RIS tags */ var TagCleaner = { + /** * public * Changes the RIS tag for an indicated tag-value pair. If more than one tag @@ -785,25 +833,26 @@ var TagCleaner = { * @param (Integer) at Index in entry of the tag-value pair to alter * @param (String[]) toTags Array of tags to change to */ - changeTag: function(entry, at, toTags) { + changeTag: function (entry, at, toTags) { var source = entry[at], byTag = entry.tags[source.tag]; //clean up "tags" list - byTag.splice(byTag.indexOf(source),1); + byTag.splice(byTag.indexOf(source), 1); if (!byTag.length) delete entry.tags[source.tag]; if (!toTags || !toTags.length) { //then we just remove - entry.splice(at,1); - } else { + entry.splice(at, 1); + } + else { source.tag = toTags[0]; //re-use the same pair for first tag if (!entry.tags[toTags[0]]) entry.tags[toTags[0]] = []; entry.tags[toTags[0]].push(source); //if we're changing to more than one tag, we need to add extras - for (var i=1, n=toTags.length; i Zotero field map this.proCiteMap = { 'Author Role': { //special case - 'actor': 'cast-member', - 'author': 'author', - 'cartographer': 'cartographer', - 'composer': 'composer', - 'composed': 'composer', - 'director': 'director', - 'directed': 'director', - 'performer': 'performer', - 'performed': 'performer', - 'producer': 'producer', - 'produced': 'producer', - 'editor': 'editor', - 'ed': 'editor', - 'edited': 'editor', + actor: 'cast-member', + author: 'author', + cartographer: 'cartographer', + composer: 'composer', + composed: 'composer', + director: 'director', + directed: 'director', + performer: 'performer', + performed: 'performer', + producer: 'producer', + produced: 'producer', + editor: 'editor', + ed: 'editor', + edited: 'editor', 'editor-in-chief': 'editor', - 'compiler': 'editor', - 'compiled': 'editor', - 'collected': 'editor', - 'assembled': 'editor', - 'presenter': 'presenter', - 'presented': 'presenter', - 'translator': 'translator', - 'translated':'translator', - 'introduction': 'contributor' + compiler: 'editor', + compiled: 'editor', + collected: 'editor', + assembled: 'editor', + presenter: 'presenter', + presented: 'presenter', + translator: 'translator', + translated: 'translator', + introduction: 'contributor' //conductor //illustrator //librettist }, 'Call Number': 'callNumber', - 'Edition': 'edition', - 'ISBN': 'ISBN', - 'Language': 'language', + Edition: 'edition', + ISBN: 'ISBN', + Language: 'language', 'Publisher Name': 'publisher', 'Series Title': 'series', 'Proceedings Title': 'proceedingsTitle', @@ -862,10 +911,10 @@ var ProCiteCleaner = new function() { 'Issue ID': 'issue', 'Issue Identification': 'issue', 'Series Volume ID': 'seriesNumber', - 'Scale': 'scale', + Scale: 'scale', 'Place of Publication': 'place', - 'Histroy': 'history', // yes, it's misspelled in their export filter - 'Size': 'artworkSize' + Histroy: 'history', // yes, it's misspelled in their export filter + Size: 'artworkSize' }; var tagValueSplit = /([A-Za-z,\s]+)\s*:\s*([\s\S]*)/; //ProCite version @@ -876,7 +925,7 @@ var ProCiteCleaner = new function() { * @param (RISReader entry) entry Entry to be cleaned up in-place * @param (Zotero.Item) item Indicates item type for proper mapping */ - this.cleanTags = function(entry, item) { + this.cleanTags = function (entry, item) { // We _must_ change some tags before mapping from notes, otherwise // there will be ambiguity var ty; @@ -889,12 +938,12 @@ var ProCiteCleaner = new function() { } - var notes = entry.tags.N1, extentOfWork, packagingMethod; + var extentOfWork, packagingMethod; //go through all the notes - for (var i=0; notes && notes.length && i=0; i--) { - + for (let i = start - 1; i >= 0; i--) { if (tag && entry[i].tag !== tag) { //different from the tags we changed previously. Don't continue return added ? added : true; } tag = entry[i].tag; - if (allowedTags.indexOf(tag) === -1) { + if (!(allowedTags.includes(tag))) { //not allowed to remap this tag - Z.debug('RIS: nothing to remap'); - return; + // Z.debug('RIS: nothing to remap'); + return false; } this._changeTag(entry, i, risTags); //don't need to adjust i, since we're traversing backwards @@ -1163,33 +1213,36 @@ var ProCiteCleaner = new function() { //we should not end up at the begining of entry, //since we will probably never be replacing TY, but just in case if (tag) return added ? added : true; + return false; }; -} +}; + /** * @singleton Fixes some EndNote bugs, makes it more conveninent to import */ -var EndNoteCleaner = new function() { +var EndNoteCleaner = new function () { + // eslint-disable-next-line lines-around-comment /** * public - * + * * @param (RISReader entry) entry Entry to be cleaned up in-place * @param (Zotero.Item) item Indicates item type for proper mapping */ - this.cleanTags = function(entry, item) { + this.cleanTags = function (entry) { // for edited books, treat authors as editors if (entry.tags.TY && entry.tags.TY[0].value == 'EDBOOK' && entry.tags.AU) { - for (var i = entry.tags.AU.length-1; i>=0; i--) { + for (let i = entry.tags.AU.length - 1; i >= 0; i--) { TagCleaner.changeTag(entry, entry.indexOf(entry.tags.AU[i]), ['A3']); } } - } + }; }; /** * @singleton Deals with some Citavi specific nuances */ -var CitaviCleaner = new function() { - this.cleanTags = function(entry, item) { +var CitaviCleaner = new function () { + this.cleanTags = function (entry) { // Citavi uses multiple H1 and H2 tags to list mutliple libraries and call // numbers for items. We can only store one, so we will transform the first // set of H1+H2 tags to DP+CN tags @@ -1199,27 +1252,27 @@ var CitaviCleaner = new function() { if (!entry.tags.H1) { // Only have a call number (maybe multiple, so take the first) - var at = entry.tags.indexOf(entry.tags.H2[0]); + let at = entry.tags.indexOf(entry.tags.H2[0]); TagCleaner.changeTag(entry, at, 'CN'); return; } if (!entry.tags.H1) { // Only have a library - var at = entry.tags.indexOf(entry.tags.H1[0]); + let at = entry.tags.indexOf(entry.tags.H1[0]); TagCleaner.changeTag(entry, at, 'DP'); } // We have pairs, so find the first set and change it - for (var i=0; i]+>/)) { //from cleanTags + } + else if (!value.match(/<[^>]+>/)) { //from cleanTags value = '

' + value.replace(/\n\n/g, '

') - .replace(/\n/g, '
') - .replace(/\t/g, '    ') - .replace(/ /g, '  ') + .replace(/\n/g, '
') + .replace(/\t/g, '    ') + .replace(/ {2}/g, '  ') + '

'; } - break; + break; case "EP": if (item.pages) { - if (item.pages.indexOf('-') == -1) { + if (!(item.pages.includes('-'))) { item.pages = item.pages + '-' + value; - } else { + } + else { item.backupNumPages = value; } value = undefined; - } else { + } + else { item.backupEndPage = value; //store this for an odd case where SP comes after EP value = undefined; } - break; + break; case "M1": //Endnote exports access date for webpages to M1 //It makes much more sense to export to Y2 @@ -1289,11 +1345,11 @@ function processTag(item, tagValue, risEntry, allowDeprecated) { item.backupAccessDate = { field: zField[0], value: dateRIStoZotero(value, zField[0]) - } + }; value = undefined; processFields = false; } - break; + break; case "M3": // This is DOI when coming from EndNote, but it can be used for // publication type as well @@ -1301,11 +1357,12 @@ function processTag(item, tagValue, risEntry, allowDeprecated) { var cleanDOI = ZU.cleanDOI(value); if (cleanDOI) { value = cleanDOI; - } else { - zField[0] = 'unknown' + } + else { + zField[0] = 'unknown'; } } - break; + break; case "VL": if (zField[0] == "accessDate") { //EndNote screws up webpage entries. VL is access year, but access date is available @@ -1318,7 +1375,7 @@ function processTag(item, tagValue, risEntry, allowDeprecated) { value = undefined; processFields = false; } - break; + break; //PY is typically less complete than other dates. We'll store it as backup case "PY": item.backupDate = { @@ -1327,35 +1384,35 @@ function processTag(item, tagValue, risEntry, allowDeprecated) { }; value = undefined; processFields = false; - break; + break; case "UR": //REFMAN places PMIDS in UR sometimes - if (value.indexOf('PM:') != -1) { + if (value.includes('PM:')) { value = 'PMID: ' + value.substr(3); zField = ['extra']; } - break; + break; } //zField based manipulations - if (processFields){ + if (processFields) { switch (zField[0]) { case "__ignore": value = undefined; - break; + break; case "backupPublicationTitle": item.backupPublicationTitle = value; value = undefined; - break; + break; case "creators": var lName = value.split(/\s*,\s*/)[0]; var fName = value.substr(lName.length).replace(/^\s*,\s*/, ''); - value = {lastName: lName, firstName:fName, creatorType:zField[1]}; + value = { lastName: lName, firstName: fName, creatorType: zField[1] }; if (!value.firstName) { //corporate delete value.firstName; value.fieldMode = 1; } - break; + break; case "date": case "accessDate": case "filingDate": @@ -1363,7 +1420,7 @@ function processTag(item, tagValue, risEntry, allowDeprecated) { case "dateEnacted": case "dateDecided": value = dateRIStoZotero(value, zField[0]); - break; + break; case "tags": //allow new lines or semicolons. Commas, might be more problematic //%K part is a hack for REFMAN exports @@ -1373,29 +1430,29 @@ function processTag(item, tagValue, risEntry, allowDeprecated) { //but it will still allow a blank tag if there is a newline or //semicolon at the begining or the end if (!value[0]) value.shift(); - if (value.length && !value[value.length-1]) value.pop(); + if (value.length && !value[value.length - 1]) value.pop(); if (!value.length) { value = undefined; } - break; + break; case "notes": - value = {note:value}; + value = { note: value }; //we can specify note title in the field mapping table if (zField[1]) { value.note = zField[1] + ': ' + value.note; } - break; + break; case "attachments": var values = value.split('\n'); var title, mimeType, url; - for (var i=0, n=values.length; i'; } - for (var i=0, n=item.unknownFields.length; i'; } if (note) { note = "The following values have no corresponding Zotero field:
" + note; - item.notes.push({note: note.trim(), tags: ['_RIS import']}); + item.notes.push({ note: note.trim(), tags: ['_RIS import'] }); } } item.unsupportedFields = undefined; item.unknownFields = undefined; - + return item.complete(); } @@ -1758,6 +1824,7 @@ function doImport() { throw e; } ); + return false; } else { return new Promise(function (resolve, reject) { @@ -1775,11 +1842,11 @@ function startImport(resolve, reject) { //prepare some configurable options if (Zotero.getHiddenPref) { - var pref = Zotero.getHiddenPref("RIS.import.ignoreUnknown"); + let pref = Zotero.getHiddenPref("RIS.import.ignoreUnknown"); if (pref != undefined) { ignoreUnknown = pref; } - var pref = Zotero.getHiddenPref("RIS.import.keepID"); + pref = Zotero.getHiddenPref("RIS.import.keepID"); if (pref === true) { degenerateImportFieldMap.ID = pref; } @@ -1795,6 +1862,7 @@ function startImport(resolve, reject) { function importNext(resolve, reject) { try { var entry; + // eslint-disable-next-line no-cond-assign while (entry = RISReader.nextEntry()) { //determine item type var itemType = exportedOptions.itemType; @@ -1811,11 +1879,12 @@ function importNext(resolve, reject) { //we allow entries without TY and just use default type if (!itemType) { var defaultType = exportedOptions.defaultItemType || DEFAULT_IMPORT_TYPE; + if (entry.tags.TY) { - Z.debug("RIS: Unknown item type: " + entry.tags.TY[0].value - + ". Defaulting to " + defaultType); - } else { - Z.debug("RIS: TY tag not specified. Defaulting to " + defaultType); + // Z.debug("RIS: Unknown item type: " + entry.tags.TY[0].value + ". Defaulting to " + defaultType); + } + else { + // Z.debug("RIS: TY tag not specified. Defaulting to " + defaultType); } itemType = defaultType; @@ -1827,20 +1896,18 @@ function importNext(resolve, reject) { CitaviCleaner.cleanTags(entry, item); var deferredEntries = []; - - for (var i=0, n=entry.length; i Date: Sat, 20 May 2023 13:55:28 -0400 Subject: [PATCH 026/125] Update DV to work properly for datasets (#3024) * Asyncify and improve short titles * Update regex * use description from schema --- Dataverse.js | 145 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 113 insertions(+), 32 deletions(-) diff --git a/Dataverse.js b/Dataverse.js index 723ac356733..2575d88c336 100644 --- a/Dataverse.js +++ b/Dataverse.js @@ -1,21 +1,21 @@ { "translatorID": "aedf3fb0-9a50-47b3-ba2f-3206552b82a9", "label": "Dataverse", - "creator": "Abe Jellinek", - "target": "^https?://(www\\.)?((open|research-|hei)?data|dvn|e?da[td]os|sodha\\.be|repositoriodedados|archive\\.data\\.jhu\\.edu|repositoriopesquisas|darus)", + "creator": "Abe Jellinek, Sebastian Karcher", + "target": "^https?://(www\\.)?((open|research-?|hei|planetary-|osna|in|bonn|borealis|lida\\.|archaeology\\.|entrepot\\.recherche\\.|archive\\.|redape\\.)?(data|e?da[td]os)|dvn|sodha\\.be|repositorio(\\.|dedados|pesquisas)|abacus\\.library\\.ubc\\.ca|dorel\\.univ-lorraine\\.fr|darus\\.uni-stuttgart\\.de|dunas\\.ua\\.pt|edmond\\.mpdl\\.mpg\\.de|keen\\.zih\\.tu-dresden\\.de|rdr\\.kuleuven\\.be|portal\\.odissei\\.nl)", "minVersion": "3.0", "maxVersion": "", "priority": 100, "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2021-08-11 18:10:33" + "lastUpdated": "2023-05-01 12:09:04" } /* ***** BEGIN LICENSE BLOCK ***** - Copyright © 2021 Abe Jellinek + Copyright © 2023 Abe Jellinek & Sebastian Karcher This file is part of Zotero. @@ -35,6 +35,9 @@ ***** END LICENSE BLOCK ***** */ +const datasetType = ZU.fieldIsValidForType('title', 'dataset') + ? 'dataset' + : 'document'; function detectWeb(doc, url) { if (!doc.querySelector('#dataverseHeader')) { @@ -42,7 +45,7 @@ function detectWeb(doc, url) { } if (url.includes('/dataset.xhtml')) { - return "document"; + return datasetType; } else if (getSearchResults(doc, true)) { return "multiple"; @@ -65,34 +68,58 @@ function getSearchResults(doc, checkOnly) { return found ? items : false; } -function doWeb(doc, url) { - if (detectWeb(doc, url) == "multiple") { - Zotero.selectItems(getSearchResults(doc, false), function (items) { - if (items) ZU.processDocuments(Object.keys(items), scrape); - }); +async function doWeb(doc, url) { + if (detectWeb(doc, url) == 'multiple') { + let items = await Zotero.selectItems(getSearchResults(doc, false)); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape(await requestDocument(url)); + } } else { - scrape(doc, url); + await scrape(doc, url); } } -function scrape(doc, url) { - var translator = Zotero.loadTranslator('web'); +async function scrape(doc, url = doc.location.href) { + let jsonLD = text(doc, 'script[type="application/ld+json"]'); + // Z.debug(jsonLD) + let schema = JSON.parse(jsonLD); + let license = schema.license; + let abstract = schema.description; + // License can be stored as a string or an object (should be a string) + if (typeof (license) == "object") { + license = schema.license.text; + } + let version = schema.version; + let translator = Zotero.loadTranslator('web'); // Embedded Metadata translator.setTranslator('951c027d-74ac-47d4-a107-9c3069ab7b48'); translator.setDocument(doc); - translator.setHandler('itemDone', function (obj, item) { + translator.setHandler('itemDone', (_obj, item) => { item.libraryCatalog = text(doc, '#breadcrumbLnk0'); + // we commonly have two colons in titles. The first one just labels the data as data + if (/^(Replication )?[Dd]ata for:/.test(item.title)) { + item.shortTitle = item.title.match(/(^(Replication )?[Dd]ata for:.*?)(:|$)/)[1]; + } + if (license) { + item.rights = ZU.cleanTags(license).trim(); + } + if (abstract) { + item.abstractNote = abstract; + } + if (version && version > 1) item.versionNumber = version; item.complete(); }); - translator.getTranslatorObject(function (trans) { - trans.itemType = 'document'; - trans.doWeb(doc, url); - }); + let em = await translator.getTranslatorObject(); + em.itemType = datasetType; + await em.doWeb(doc, url); } + + /** BEGIN TEST CASES **/ var testCases = [ { @@ -100,7 +127,7 @@ var testCases = [ "url": "https://dataverse.harvard.edu/dataset.xhtml?persistentId=doi:10.7910/DVN/3OUPKS", "items": [ { - "itemType": "document", + "itemType": "dataset", "title": "Survey seed production and seed contract", "creators": [ { @@ -110,11 +137,12 @@ var testCases = [ } ], "date": "2021-08-04", + "DOI": "10.7910/DVN/3OUPKS", "abstractNote": "The data is collected as part of a seed contract experiment conducted in Telengana state, India", - "extra": "Type: dataset\nDOI: 10.7910/DVN/3OUPKS", "language": "en", "libraryCatalog": "Harvard Dataverse", - "publisher": "Harvard Dataverse", + "repository": "Harvard Dataverse", + "rights": "CC0", "url": "https://dataverse.harvard.edu/dataset.xhtml?persistentId=doi:10.7910/DVN/3OUPKS", "attachments": [ { @@ -140,9 +168,10 @@ var testCases = [ { "type": "web", "url": "https://datos.uchile.cl/dataset.xhtml?persistentId=doi:10.34691/FK2/UMWJDU", + "detectedItemType": "dataset", "items": [ { - "itemType": "document", + "itemType": "dataset", "title": "Replicar los datos para: Genetic diversity in a restricted-dispersal kissing bug: The Center-Periphery Hypothesis halfway", "creators": [ { @@ -172,11 +201,13 @@ var testCases = [ } ], "date": "2021-07-19", + "DOI": "10.34691/FK2/UMWJDU", "abstractNote": "The center-periphery hypothesis (CPH) postulates that populations close to the center of a species’ distribution will exhibit higher genetic diversity and lower genetic differentiation than populations located at the edge of the distribution. The center of a species distribution might represent an optimum for the environmental factors influencing the species absolute fitness and therefore, genetic diversity. In species with wide distribution, the geographical variation of biotic and abiotic variables is crucial to understand the underlying mechanisms of the CPH. We evaluated the CPH and specifically tested which environmental variables better explained the patterns of genetic diversity in the kissing-bug Mepraia spinolai, one of the main wild vectors of Chagas disease in southern South America, distributing across three Mediterranean climatic ecoregions in Chile. We analyzed 2380 neutral Single Nucleotide Polymorphisms (SNP) to estimate genetic diversity. The mean winter temperature, the mean summer temperature, vegetation cover, population abundance, proportion of winged individuals and female abdomen area were measured for each kissing bug population to construct a model. Lower genetic diversity was detected in populations at the edge of the distribution compared to those in the center. However, genetic differentiation was not higher in the periphery. Genetic diversity was related to climatic and biological variables; there was a positive relationship with mean winter temperature and a negative association with mean summer temperature and body size. These results partially support the CPH and identify biotic (abdomen area) and abiotic (winter/summer temperatures) factors that would affect genetic diversity in this restricted-dispersal species of epidemiological relevance.", - "extra": "Type: dataset\nDOI: 10.34691/FK2/UMWJDU", + "extra": "Type: dataset", "language": "en", - "libraryCatalog": "Universidad de Chile", - "publisher": "Universidad de Chile", + "libraryCatalog": "Repositorio de datos de investigación de la Universidad de Chile", + "repository": "Repositorio de datos de investigación de la Universidad de Chile", + "rights": "CC0", "shortTitle": "Replicar los datos para", "url": "https://datos.uchile.cl/dataset.xhtml?persistentId=doi:10.34691/FK2/UMWJDU", "attachments": [ @@ -205,7 +236,7 @@ var testCases = [ "url": "https://www.sodha.be/dataset.xhtml?persistentId=doi:10.34934/DVN/HEABHJ", "items": [ { - "itemType": "document", + "itemType": "dataset", "title": "World Value Surveys 1981", "creators": [ { @@ -215,11 +246,13 @@ var testCases = [ } ], "date": "2021-07-30", - "abstractNote": "The series is designed to enable a crossnational comparison of values and norms on a wide variety of topics and to monitor changes in values and at...", - "extra": "Type: dataset\nDOI: 10.34934/DVN/HEABHJ", + "DOI": "10.34934/DVN/HEABHJ", + "abstractNote": "The series is designed to enable a crossnational comparison of values and norms on a wide variety of topics and to monitor changes in values and attitudes across the world in 1981.", + "extra": "Type: dataset", "language": "en", "libraryCatalog": "Social Sciences and Digital Humanities Archive – SODHA", - "publisher": "Social Sciences and Digital Humanities Archive – SODHA", + "repository": "Social Sciences and Digital Humanities Archive – SODHA", + "rights": "This work is licensed under a Creative Commons Attribution 4.0 International License (CC-BY).", "url": "https://www.sodha.be/dataset.xhtml?persistentId=doi:10.34934/DVN/HEABHJ", "attachments": [ { @@ -242,7 +275,7 @@ var testCases = [ "url": "https://data.worldagroforestry.org/dataset.xhtml?persistentId=doi:10.34725/DVN/NGGXCZ", "items": [ { - "itemType": "document", + "itemType": "dataset", "title": "Replication Data for: Agroforestry governance for operationalising the landscape approach: connecting conservation and farming actors", "creators": [ { @@ -307,11 +340,12 @@ var testCases = [ } ], "date": "2021-08-05", + "DOI": "10.34725/DVN/NGGXCZ", "abstractNote": "The expansion and intensifcation of agriculture as well as the associated land clearing are threatening both biodiversity and human wellbeing in tr...", - "extra": "Type: dataset\nDOI: 10.34725/DVN/NGGXCZ", "language": "en", "libraryCatalog": "World Agroforestry - Research Data Repository", - "publisher": "World Agroforestry - Research Data Repository", + "repository": "World Agroforestry - Research Data Repository", + "rights": "This dataset is made available under the Creative Commons Attribution 4.0 International (CC-BY-4.0). The license allows you, the user, to copy and redistribute the material in any medium or format and/or transform, and build upon the material\r\nfor any purpose, even commercially.", "shortTitle": "Replication Data for", "url": "https://data.worldagroforestry.org/dataset.xhtml?persistentId=doi:10.34725/DVN/NGGXCZ", "attachments": [ @@ -332,6 +366,53 @@ var testCases = [ "seeAlso": [] } ] + }, + { + "type": "web", + "url": "https://dataverse.no/dataset.xhtml?persistentId=doi:10.18710/GUX2O8", + "detectedItemType": "dataset", + "items": [ + { + "itemType": "dataset", + "title": "Replication Data for: Spatial changes in gas transport and sediment stiffness influenced by regional stress: observations from piezometer data along the Vestnesa Ridge, eastern Fram Strait", + "creators": [ + { + "firstName": "Andreia", + "lastName": "Plaza-Faverola", + "creatorType": "author" + }, + { + "firstName": "Nabil", + "lastName": "Sultan", + "creatorType": "author" + } + ], + "date": "2023-04-11", + "DOI": "10.18710/GUX2O8", + "abstractNote": "This database comprises piezometer pore-pressure and temperature data, analyses from Calypso sediment cores (grain sizes and logs) and results from...", + "extra": "Type: dataset", + "language": "en", + "libraryCatalog": "DataverseNO", + "repository": "DataverseNO", + "rights": "http://creativecommons.org/publicdomain/zero/1.0", + "shortTitle": "Replication Data for", + "url": "https://dataverse.no/dataset.xhtml?persistentId=doi:10.18710/GUX2O8", + "versionNumber": "2", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [ + { + "tag": "Earth and Environmental Sciences" + } + ], + "notes": [], + "seeAlso": [] + } + ] } ] /** END TEST CASES **/ From 5556c3173b02d2f6a1a06ceba44728d6af479f58 Mon Sep 17 00:00:00 2001 From: Sebastian Karcher Date: Sat, 20 May 2023 13:55:57 -0400 Subject: [PATCH 027/125] Add Databrary Translator (#3020) --- Databrary.js | 184 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 Databrary.js diff --git a/Databrary.js b/Databrary.js new file mode 100644 index 00000000000..ae71bf0b97f --- /dev/null +++ b/Databrary.js @@ -0,0 +1,184 @@ +{ + "translatorID": "45ece855-7303-41d2-8c9f-1151f684943c", + "label": "Databrary", + "creator": "Sebastian Karcher", + "target": "^https?://nyu\\.databrary\\.org/(volume|search)", + "minVersion": "5.0", + "maxVersion": "", + "priority": 100, + "inRepository": true, + "translatorType": 4, + "browserSupport": "gcsibv", + "lastUpdated": "2023-04-19 16:49:05" +} + +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2023 Sebastian Karcher + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + + +const datasetType = ZU.fieldIsValidForType('title', 'dataset') + ? 'dataset' + : 'document'; + +function detectWeb(doc, url) { + if (url.includes('/volume/')) { + return datasetType; + } + else if (url.includes("/search?q=")) { + Z.monitorDOMChanges(doc.querySelector('body')); + if (getSearchResults(doc, true)) return 'multiple'; + } + return false; +} + +function getSearchResults(doc, checkOnly) { + var items = {}; + var found = false; + var rows = doc.querySelectorAll('h3.search-volume-result-title>a'); + for (let row of rows) { + let href = row.href; + let title = ZU.trimInternal(row.textContent); + if (!href || !title) continue; + if (checkOnly) return true; + found = true; + items[href] = title; + } + return found ? items : false; +} + +async function doWeb(doc, url) { + if (detectWeb(doc, url) == 'multiple') { + let items = await Zotero.selectItems(getSearchResults(doc, false)); + if (!items) return; + for (let url of Object.keys(items)) { + // page loads via javascript, so geting the object wont' work + url = url.replace("https://nyu.databrary.org/volume", "https://nyu.databrary.org/api/volume"); + await scrapeJSON(await requestText(url)); + } + } + else { + await scrape(doc, url); + } +} + +async function scrapeJSON(text) { + var dbJSON = JSON.parse(text); + let doi = dbJSON.doi; + let risURL = `https://data.datacite.org/application/x-research-info-systems/${doi}`; + let id = dbJSON.id; + let risText = await requestText(risURL); + let translator = Zotero.loadTranslator('import'); + translator.setTranslator('32d59d2d-b65a-4da4-b0a3-bdd3cfb979e7'); // RIS + translator.setString(risText); + translator.setHandler('itemDone', (_obj, item) => { + item.itemType = "dataset"; + item.attachments.push({ + title: 'Snapshot', + url: `http://databrary.org/volume/${id}`, + mimeType: 'text/html' + }); + + item.complete(); + }); + await translator.translate(); +} + +async function scrape(doc) { + let DOI = attr(doc, 'p.panel-overview-volume-citation>a', 'href'); + DOI = DOI.replace(/https?:\/\/doi.org/, ""); + let risURL = `https://data.datacite.org/application/x-research-info-systems/${DOI}`; + // Z.debug(risURL) + + let risText = await requestText(risURL); + // Z.debug(risText); + let translator = Zotero.loadTranslator('import'); + translator.setTranslator('32d59d2d-b65a-4da4-b0a3-bdd3cfb979e7'); // RIS + translator.setString(risText); + translator.setHandler('itemDone', (_obj, item) => { + item.attachments.push({ + title: 'Snapshot', + document: doc + }); + + item.complete(); + }); + await translator.translate(); +} + + +/** BEGIN TEST CASES **/ +var testCases = [ + { + "type": "web", + "url": "https://nyu.databrary.org/volume/1322", + "defer": true, + "detectedItemType": "dataset", + "items": [ + { + "itemType": "dataset", + "title": "Moving language: Mothers’ verbs correspond to infants’ real-time locomotion", + "creators": [ + { + "lastName": "West", + "firstName": "Kelsey Louise", + "creatorType": "author" + }, + { + "lastName": "Tamis-LeMonda", + "firstName": "Catherine", + "creatorType": "author" + }, + { + "lastName": "Adolph", + "firstName": "Karen", + "creatorType": "author" + } + ], + "date": "2021", + "DOI": "10.17910/B7.1322", + "abstractNote": "How do infants learn language? Infants can only learn the words that they hear. We tested whether infants’ actions affect the words that caregivers say—specifically whether infant locomotion influences caregivers’ language about locomotion. Compared to crawling infants, walkers travel greater distances (Adolph, et al, 2012). Does enhanced locomotion in walkers influence the verbs that caregivers say? We hypothesized that walking creates new opportunities for verb learning. To disentangle locomotor ability from age, we observed same-aged crawlers and walkers (16 13-month-old crawlers and 16 13-month-old walkers) and an older group of walkers (16 18-month-olds) during two hours of activity at home. Mothers’ language was transcribed verbatim. We then identified each “locomotor verb” (e.g., “come,” “bring”) that mothers said, and each bout of infant crawling and walking. Walkers’ enhanced locomotion indeed opened new opportunities for verb learning. Although mothers’ language overall was more frequent to older compared to younger infants, their locomotor verbs were more frequent to walkers than to crawlers. Preliminary findings show that caregivers directed more utterances to 18-month-olds (M = 2,006.00, SD = 579.02) compared to 13-month-old crawlers (M = 1,553.75, SD = 728.42) and walkers (M = 1,363.50, SD = 619.29), F (2, 31) = 3.23, p = .052. Notably, caregivers directed twice as many locomotor verbs to 13- and 18-month-old walkers (M = 53.13, SD = 15.40; M = 53.00, SD = 25.14, respectively) compared with 13-month-old crawlers (M = 25.25, SD = 12.87), F (2, 31) = 5.46, p = .01. Moreover, mothers’ locomotor verbs were related to infants’ moment-to-moment locomotion: Infants who moved more frequently received more locomotor verbs compared to infants who moved less, r(26) = .42, p = .035. Findings indicate that locomotor development leads to more advanced forms of infant activity, which consequently prompts caregivers to use more advanced language.", + "libraryCatalog": "Databrary", + "repository": "Databrary", + "shortTitle": "Moving language", + "url": "http://databrary.org/volume/1322", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://nyu.databrary.org/search?q=mothers", + "defer": true, + "detectedItemType": "multiple", + "items": "multiple" + } +] +/** END TEST CASES **/ From a731ec5706314a457b0934dde7fe35a2b65abc74 Mon Sep 17 00:00:00 2001 From: Sebastian Karcher Date: Sat, 20 May 2023 14:14:18 -0400 Subject: [PATCH 028/125] Add dataset import support to RDF (#3029) Export still todo --- RDF.js | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/RDF.js b/RDF.js index 75e9b8d7ed5..f39e9a5cf92 100644 --- a/RDF.js +++ b/RDF.js @@ -12,7 +12,7 @@ }, "inRepository": true, "translatorType": 1, - "lastUpdated": "2022-09-22 02:12:48" + "lastUpdated": "2023-04-24 15:51:01" } /* @@ -39,6 +39,9 @@ ***** END LICENSE BLOCK ***** */ +const datasetType = ZU.fieldIsValidForType('title', 'dataset') + ? 'dataset' + : 'document'; function detectImport() { // Make sure there are actually nodes @@ -444,8 +447,7 @@ function detectType(newItem, node, ret) { t.so = 'artwork'; break; case 'datacatalog': case 'dataset': - t.so = 'journalArticle'; break; // until dataset gets implemented - + t.so = datasetType; break; // specials cases case "article": // choose between journal, newspaper, and magazine articles @@ -532,8 +534,10 @@ function detectType(newItem, node, ret) { case 'conferenceposter': t.dc = 'conferencePaper'; break; + case 'dataset': + t.dc = datasetType; + break; case 'article': // from http://www.idealliance.org/specifications/prism/specifications/prism-controlled-vocabularies/prism-12-controlled-vocabularies - case 'dataset': // until dataset gets implemented case 'journalitem': case 'journalarticle': case 'submittedjournalarticle': @@ -616,7 +620,7 @@ function detectType(newItem, node, ret) { // very broad t.dcGuess = 'journalArticle'; break; - // collection, dataset, interactiveresource, physicalobject, + // collection, interactiveresource, physicalobject, // service } } @@ -644,10 +648,12 @@ function detectType(newItem, node, ret) { case 'conferenceposter': t.eprints = 'conferencePaper'; break; + case 'dataset': + t.eprints = datasetType; + break; case 'journalitem': case 'journalarticle': case 'submittedjournalarticle': - case 'dataset': // map to dataset once we have it as item type case 'article': t.eprints = 'journalArticle'; break; @@ -1220,8 +1226,8 @@ function importItem(newItem, node) { // type let type = getFirstResults(node, [n.dc + "type", n.dc1_0 + "type", n.dcterms + "type"], true); - /** CUSTOM ITEM TYPE -- Currently only Dataset **/ - if (type && (type.toLowerCase() == "dataset" || type.toLowerCase() == "datacatalog")) { + /** CUSTOM ITEM TYPE -- Keeping for <6.0.24 clients w/o dataset support **/ + if (datasetType != "dataset" && type && (type.toLowerCase() == "dataset" || type.toLowerCase() == "datacatalog")) { if (newItem.extra) { newItem.extra += "\nType: dataset"; } From 1af44d1eaec8808339b851e1ba0678577833179d Mon Sep 17 00:00:00 2001 From: Sebastian Karcher Date: Sat, 20 May 2023 14:29:29 -0400 Subject: [PATCH 029/125] SEP update & make work for archive pages (#3034) https://forums.zotero.org/discussion/103528/stanford-encyclopedia-of-philosophy-oddities#latest --- Stanford Encyclopedia of Philosophy.js | 262 ++++++++++++++++++------- 1 file changed, 195 insertions(+), 67 deletions(-) diff --git a/Stanford Encyclopedia of Philosophy.js b/Stanford Encyclopedia of Philosophy.js index db9505fd0d3..e1e7cb2b195 100644 --- a/Stanford Encyclopedia of Philosophy.js +++ b/Stanford Encyclopedia of Philosophy.js @@ -2,22 +2,20 @@ "translatorID": "5aabfa6e-79e6-4791-a9d2-46c9cb137561", "label": "Stanford Encyclopedia of Philosophy", "creator": "Sebastian Karcher", - "target": "^https?://plato\\.stanford\\.edu/(entries|search)", + "target": "^https?://plato\\.stanford\\.edu/(archives/[a-z]{3}\\d{4}/)?(entries|search)", "minVersion": "2.1", "maxVersion": "", "priority": 100, "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2014-07-05 10:41:59" + "lastUpdated": "2023-05-04 15:33:21" } /* ***** BEGIN LICENSE BLOCK ***** - Copyright © 2011 Sebastian Karcher and the Center for History and New Media - George Mason University, Fairfax, Virginia, USA - http://zotero.org + Copyright © 2011-2023 Sebastian Karcher and Zotero This file is part of Zotero. @@ -38,66 +36,77 @@ */ function detectWeb(doc, url) { - if (url.match(/\/search\//)) return "multiple"; - if (url.match(/\entries\//)) return "bookSection"; + if (url.includes("/search/") && getSearchResults(doc, true)) return "multiple"; + else if (url.includes("/entries/")) return "bookSection"; + return false; } -function doWeb(doc, url){ +function getSearchResults(doc, checkOnly) { + var items = {}; + var found = false; + var rows = doc.querySelectorAll('div.result_title > a'); + for (let row of rows) { + let href = row.href; + let title = ZU.trimInternal(row.textContent); + if (!href || !title) continue; + if (checkOnly) return true; + found = true; + items[href] = title; + } + return found ? items : false; +} - var articles = new Array(); - if (detectWeb(doc, url) == "multiple") { - var items = {}; - var titles = doc.evaluate('//div[@class="result_title"]/a', doc, null, XPathResult.ANY_TYPE, null); - var title; - while (title = titles.iterateNext()) { - items[title.href] = title.textContent; +async function doWeb(doc, url) { + if (detectWeb(doc, url) == 'multiple') { + let items = await Zotero.selectItems(getSearchResults(doc, false)); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape(await requestDocument(url)); } - Zotero.selectItems(items, function (items) { - if (!items) { - return true; - } - for (var i in items) { - articles.push(i); - } - Zotero.Utilities.processDocuments(articles, scrape); - }); - } else { - scrape(doc, url); + } + else { + await scrape(doc, url); } } -// help function -function scrape(doc, url){ + +async function scrape(doc, url = doc.location.href) { //get abstract and tags from article plage - //the xpaths aren't great , but seem reliable across pages - var abs = ZU.xpathText(doc,'//div[@id="preamble"]/p').replace(/\n/g, ""); - var tags = ZU.xpathText(doc, '//div[@id="article-content"]//h2[a[@name="Rel" or @id="Rel"]]/following-sibling::p'); + var abs = text(doc, 'div#preamble>p').replace(/\n/g, ""); + var tags = text(doc, 'div#related-entries >p'); + // Z.debug(tags); if (tags) tags = tags.replace(/\n/g, "").split(/\|/); - for (i in tags){ - tags[i] = ZU.trimInternal(tags[i]) - } + for (let i = 0; i < tags.length; i++) { + tags[i] = ZU.trimInternal(tags[i]); + } //get BibTex Link - var bibtexurl = url.replace(/entries\//,"cgi-bin/encyclopedia/archinfo.cgi?entry=").replace(/\/(index\.html)?(#.+)?$/, ""); - //Z.debug(bibtexurl) - Zotero.Utilities.HTTP.doGet(bibtexurl, function (text) { - //Z.debug(text) - //remove line breaks, then match match the bibtex, then remove the odd /t in it. - bibtex = text.replace(/\n/g, "").match(/
.+<\/pre>/)[0].replace(/\/t/g, "")
-	//Zotero.debug(bibtex)
-
-		var translator = Zotero.loadTranslator("import");
-		translator.setTranslator("9cb70025-a888-4a29-a210-93ec52da40d4");
-		translator.setString(bibtex);
-		translator.setHandler("itemDone", function(obj, item) {
-			if (abs) item.abstractNote = abs;
-			if (tags) item.tags = tags;
-			item.attachments = [{url:item.url, title: "SEP - Snapshot", mimeType: "text/html"}];
-			item.complete();
-		});	
-		translator.translate();
+	var bibtexUrl = url.replace(/entries\//, "cgi-bin/encyclopedia/archinfo.cgi?entry=").replace(/\/(index\.html)?(#.+)?$/, "");
+	if (bibtexUrl.includes("/archives/")) {
+		// we're on archive page
+		let archive = bibtexUrl.match(/\/(archives\/[a-z]{3}\d{4})/)[1];
+		bibtexUrl = bibtexUrl.replace(archive, "/") + "&" + archive.replace("s/", "=");
+		// Z.debug(bibtexUrl);
+	}
+	
+	// Z.debug(bibtexUrl)
+	let bibtex = await requestText(bibtexUrl);
+	// remove line breaks, then match match the bibtex, then remove the odd /t in it.
+	bibtex = bibtex.replace(/\n/g, "").match(/
.+<\/pre>/)[0].replace(/\/t/g, "");
+	// Zotero.debug(bibtex)
+	var translator = Zotero.loadTranslator("import");
+	translator.setTranslator("9cb70025-a888-4a29-a210-93ec52da40d4");
+	translator.setString(bibtex);
+	translator.setHandler("itemDone", (_obj, item) => {
+		if (abs) item.abstractNote = abs;
+		if (tags) item.tags = tags;
+		item.attachments = [{ document: doc, title: "SEP - Snapshot", mimeType: "text/html" }];
+		item.complete();
 	});
-}/** BEGIN TEST CASES **/
+	await translator.translate();
+}
+
+/** BEGIN TEST CASES **/
 var testCases = [
 	{
 		"type": "web",
@@ -123,14 +132,91 @@ var testCases = [
 						"creatorType": "editor"
 					}
 				],
-				"date": "2015",
-				"abstractNote": "Plato (429?–347 B.C.E.) is, by any reckoning, one of the mostdazzling writers in the Western literary tradition and one of the mostpenetrating, wide-ranging, and influential authors in the history ofphilosophy. An Athenian citizen of high status, he displays in hisworks his absorption in the political events and intellectual movementsof his time, but the questions he raises are so profound and thestrategies he uses for tackling them so richly suggestive andprovocative that educated readers of nearly every period have in someway been influenced by him, and in practically every age there havebeen philosophers who count themselves Platonists in some importantrespects. He was not the first thinker or writer to whom the word“philosopher” should be applied. But he was soself-conscious about how philosophy should be conceived, and what itsscope and ambitions properly are, and he so transformed theintellectual currents with which he grappled, that the subject ofphilosophy, as it is often conceived—a rigorous and systematicexamination of ethical, political, metaphysical, and epistemologicalissues, armed with a distinctive method—can be called hisinvention. Few other authors in the history of Western philosophy approximatehim in depth and range: perhaps only Aristotle (who studied with him),Aquinas, and Kant would be generally agreed to be of the same rank.",
+				"date": "2022",
+				"abstractNote": "Plato (429?–347 B.C.E.) is, by any reckoning, one of the mostdazzling writers in the Western literary tradition and one of the mostpenetrating, wide-ranging, and influential authors in the history ofphilosophy. An Athenian citizen of high status, he displays in hisworks his absorption in the political events and intellectualmovements of his time, but the questions he raises are so profound andthe strategies he uses for tackling them so richly suggestive andprovocative that educated readers of nearly every period have in someway been influenced by him, and in practically every age there havebeen philosophers who count themselves Platonists in some importantrespects. He was not the first thinker or writer to whom the word“philosopher” should be applied. But he was soself-conscious about how philosophy should be conceived, and what itsscope and ambitions properly are, and he so transformed theintellectual currents with which he grappled, that the subject ofphilosophy, as it is often conceived—a rigorous and systematicexamination of ethical, political, metaphysical, and epistemologicalissues, armed with a distinctive method—can be called hisinvention. Few other authors in the history of Western philosophyapproximate him in depth and range: perhaps only Aristotle (whostudied with him), Aquinas, and Kant would be generally agreed to beof the same rank.",
 				"bookTitle": "The Stanford Encyclopedia of Philosophy",
-				"edition": "Spring 2015",
+				"edition": "Spring 2022",
 				"itemID": "sep-plato",
 				"libraryCatalog": "Stanford Encyclopedia of Philosophy",
 				"publisher": "Metaphysics Research Lab, Stanford University",
-				"url": "https://plato.stanford.edu/archives/spr2015/entries/plato/",
+				"url": "https://plato.stanford.edu/archives/spr2022/entries/plato/",
+				"attachments": [
+					{
+						"title": "SEP - Snapshot",
+						"mimeType": "text/html"
+					}
+				],
+				"tags": [
+					{
+						"tag": "Aristotle"
+					},
+					{
+						"tag": "Plato: ethics and politics in The Republic"
+					},
+					{
+						"tag": "Socrates"
+					},
+					{
+						"tag": "Socratic Dialogues"
+					},
+					{
+						"tag": "abstract objects"
+					},
+					{
+						"tag": "education, philosophy of"
+					},
+					{
+						"tag": "epistemology"
+					},
+					{
+						"tag": "metaphysics"
+					},
+					{
+						"tag": "religion: and morality"
+					}
+				],
+				"notes": [],
+				"seeAlso": []
+			}
+		]
+	},
+	{
+		"type": "web",
+		"url": "https://plato.stanford.edu/archives/win2022/entries/logic-classical/",
+		"items": [
+			{
+				"itemType": "bookSection",
+				"title": "Classical Logic",
+				"creators": [
+					{
+						"firstName": "Stewart",
+						"lastName": "Shapiro",
+						"creatorType": "author"
+					},
+					{
+						"firstName": "Teresa",
+						"lastName": "Kouri Kissel",
+						"creatorType": "author"
+					},
+					{
+						"firstName": "Edward N.",
+						"lastName": "Zalta",
+						"creatorType": "editor"
+					},
+					{
+						"firstName": "Uri",
+						"lastName": "Nodelman",
+						"creatorType": "editor"
+					}
+				],
+				"date": "2022",
+				"abstractNote": "Typically, a logic consists of a formal or informal languagetogether with a deductive system and/or a model-theoretic semantics.The language has components that correspond to a part of a naturallanguage like English or Greek. The deductive system is to capture,codify, or simply record arguments that are validfor the given language, and the semantics is to capture, codify, orrecord the meanings, or truth-conditions for at least part of thelanguage.",
+				"bookTitle": "The Stanford Encyclopedia of Philosophy",
+				"edition": "Winter 2022",
+				"itemID": "sep-logic-classical",
+				"libraryCatalog": "Stanford Encyclopedia of Philosophy",
+				"publisher": "Metaphysics Research Lab, Stanford University",
+				"url": "https://plato.stanford.edu/archives/win2022/entries/logic-classical/",
 				"attachments": [
 					{
 						"title": "SEP - Snapshot",
@@ -138,15 +224,57 @@ var testCases = [
 					}
 				],
 				"tags": [
-					"Aristotle",
-					"Plato: ethics and politics in The Republic",
-					"Socrates",
-					"Socratic Dialogues",
-					"abstract objects",
-					"education, philosophy of",
-					"epistemology",
-					"metaphysics",
-					"religion: and morality"
+					{
+						"tag": "logic: free"
+					},
+					{
+						"tag": "logic: infinitary"
+					},
+					{
+						"tag": "logic: intuitionistic"
+					},
+					{
+						"tag": "logic: linear"
+					},
+					{
+						"tag": "logic: modal"
+					},
+					{
+						"tag": "logic: paraconsistent"
+					},
+					{
+						"tag": "logic: relevance"
+					},
+					{
+						"tag": "logic: second-order and higher-order"
+					},
+					{
+						"tag": "logic: substructural"
+					},
+					{
+						"tag": "logic: temporal"
+					},
+					{
+						"tag": "logical consequence"
+					},
+					{
+						"tag": "logical form"
+					},
+					{
+						"tag": "logical truth"
+					},
+					{
+						"tag": "model theory"
+					},
+					{
+						"tag": "model theory: first-order"
+					},
+					{
+						"tag": "paradox: Skolem’s"
+					},
+					{
+						"tag": "proof theory: development of"
+					}
 				],
 				"notes": [],
 				"seeAlso": []
@@ -154,4 +282,4 @@ var testCases = [
 		]
 	}
 ]
-/** END TEST CASES **/
\ No newline at end of file
+/** END TEST CASES **/

From ab8a83ebba9a165ad44ace10b24c582c1bd52424 Mon Sep 17 00:00:00 2001
From: Sebastian Karcher 
Date: Sat, 20 May 2023 14:36:41 -0400
Subject: [PATCH 030/125] Add CFF export (#3032)

---
 CFF.js | 113 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 113 insertions(+)
 create mode 100644 CFF.js

diff --git a/CFF.js b/CFF.js
new file mode 100644
index 00000000000..8ce6213d1bc
--- /dev/null
+++ b/CFF.js
@@ -0,0 +1,113 @@
+{
+	"translatorID": "e782b521-99ed-47c7-b021-62351a0a4f91",
+	"label": "CFF",
+	"creator": "Sebastian Karcher",
+	"target": "cff",
+	"minVersion": "5.0",
+	"maxVersion": "",
+	"priority": 100,
+	"inRepository": true,
+	"translatorType": 2,
+	"lastUpdated": "2023-05-04 13:21:10"
+}
+
+/*
+	***** BEGIN LICENSE BLOCK *****
+
+	Copyright © 2023 Sebastian Karcher
+
+	This file is part of Zotero.
+
+	Zotero is free software: you can redistribute it and/or modify
+	it under the terms of the GNU Affero General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	Zotero is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+	GNU Affero General Public License for more details.
+
+	You should have received a copy of the GNU Affero General Public License
+	along with Zotero. If not, see .
+
+	***** END LICENSE BLOCK *****
+*/
+
+function writeKeywords(tags) {
+	if (!tags.length) return false;
+	let keywords = "\n";
+	for (let tag of tags) {
+		keywords += "  - " + tag.tag + "\n";
+	}
+	return keywords.replace(/\\n$/, "");
+}
+
+function writeDOI(itemDOI) {
+	if (!itemDOI) return false;
+	let doi = "\n  - type: doi\n    value: " + itemDOI + "\n";
+	return doi;
+}
+
+function writeAuthors(itemCreators) {
+	let itemAuthors = [];
+	for (let creator of itemCreators) {
+		if (creator.creatorType == "author" || creator.creatorType == "programmer") {
+			itemAuthors.push(creator);
+		}
+	}
+	if (!itemAuthors.length) return false;
+	let authors = "\n";
+	for (let author of itemAuthors) {
+		authors += "  - family-names: " + author.lastName + "\n";
+		if (author.firstName) {
+			authors += "    given-names: " + author.firstName + "\n";
+		}
+	}
+	return authors;
+}
+
+function doExport() {
+	var item;
+	Zotero.write('# This CITATION.cff file was generated with Zotero.\n');
+	// eslint-disable-next-line no-cond-assign
+	while (item = Zotero.nextItem()) {
+		// Only use for dataset and software
+		if (item.itemType != "dataset" && item.itemType != "computerProgram") {
+			continue;
+		}
+		var cff = {};
+		cff.title = " >-\n  " + item.title;
+		cff.abstract = item.abstractNote;
+		cff.type = item.itemType == 'dataset' ? 'dataset' : 'software';
+		cff.license = item.rights;
+		cff.version = item.versionNumber;
+		cff.url = item.url;
+		cff.keywords = writeKeywords(item.tags);
+		cff.authors = writeAuthors(item.creators);
+		if (item.date) {
+			cff["date-released"] = ZU.strToISO(item.date);
+		}
+		// get DOI from Extra for software; this will stop running automatically once software supports DOI
+		if (!ZU.fieldIsValidForType('DOI', item.itemType) && /^doi:/i.test(item.extra)) {
+			item.DOI = ZU.cleanDOI(item.extra);
+		}
+		cff.identifiers = writeDOI(item.DOI);
+
+		Zotero.write(`\ncff-version: 1.2.0\nmessage: >-\n  If you use this ${cff.type}, please cite it using the metadata from this file.\n`);
+		for (let field in cff) {
+			if (!cff[field]) continue;
+			if (field == "authors" || field == "keywords") {
+				Zotero.write(field + ": " + cff[field]);
+			}
+			else {
+				Zotero.write(field + ": " + cff[field] + "\n");
+			}
+		}
+	}
+}
+
+/** BEGIN TEST CASES **/
+var testCases = [
+]
+/** END TEST CASES **/

From e458dc3402cb2d7a788aeb0c4758035492182623 Mon Sep 17 00:00:00 2001
From: bfahrenfort <59982409+bfahrenfort@users.noreply.github.com>
Date: Thu, 25 May 2023 23:14:14 -0500
Subject: [PATCH 031/125] Lexis+: Fix bug with word capture for "act" (#3038)

---
 Lexis+.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Lexis+.js b/Lexis+.js
index f7357fffcf4..98f68dfcb73 100644
--- a/Lexis+.js
+++ b/Lexis+.js
@@ -9,7 +9,7 @@
 	"inRepository": true,
 	"translatorType": 4,
 	"browserSupport": "gcsibv",
-	"lastUpdated": "2023-04-10 03:15:48"
+	"lastUpdated": "2023-05-26 04:11:53"
 }
 
 /*
@@ -40,7 +40,7 @@ function detectWeb(doc, _url) {
 		return "multiple";
 	}
 	else if (/[a-zA-Z. ]+\s§\s\d+/.test(doc.title)
-	|| /act/i.test(doc.title)
+	|| /\W(acts?)(\W|$)/i.test(doc.title) // Match: The Airports Acts, Civil Rights Act of 1865
 	|| /p\.l\./i.test(doc.title)) { // Match: ... Tex. Bus. & Com. Code § 26.01 ...
 		return "statute";
 	}

From 038892b593ad2f477cd64dc4745a036ae406d076 Mon Sep 17 00:00:00 2001
From: zoe-translates <116055375+zoe-translates@users.noreply.github.com>
Date: Tue, 6 Jun 2023 13:12:35 +0800
Subject: [PATCH 032/125] Trove: Update API key and fix HTTP 401 Unauthorized
 error. (#3044)

The previous static API key is not working anymore. The current Trove
web app computes the API key using a cookie, making it vary across
sessions.

To fix this, the translator now computes the key using the same
algorithm.
---
 Trove.js | 41 +++++++++++++++++++++++++++++++++--------
 1 file changed, 33 insertions(+), 8 deletions(-)

diff --git a/Trove.js b/Trove.js
index 16f43373750..8e1dfdfcf8e 100644
--- a/Trove.js
+++ b/Trove.js
@@ -9,7 +9,7 @@
 	"inRepository": true,
 	"translatorType": 4,
 	"browserSupport": "gcsibv",
-	"lastUpdated": "2021-06-23 05:56:37"
+	"lastUpdated": "2023-06-06 04:17:30"
 }
 
 /*
@@ -103,7 +103,9 @@ function doWeb(doc, url) {
 			if (!items) return;
 
 			for (var i in items) {
-				scrape(null, i);
+				// Pass the current document as context for cookie
+				// retrieval and API-key computation.
+				scrape(null, i, doc/* docContext */);
 			}
 		});
 	}
@@ -113,12 +115,12 @@ function doWeb(doc, url) {
 }
 
 
-function scrape(doc, url) {
+function scrape(doc, url, docContext = doc) {
 	if (url.includes('/newspaper/article/')) {
 		scrapeNewspaper(doc, url);
 	}
 	else {
-		scrapeWork(doc, url);
+		scrapeWork(doc, url, docContext);
 	}
 }
 
@@ -342,7 +344,7 @@ function cleanEdition(text) {
 	}
 }
 
-function scrapeWork(doc, url) {
+function scrapeWork(doc, url, docContext) {
 	var thumbnailURL;
 
 	var workID = url.match(/\/work\/([0-9]+)/)[1];
@@ -403,12 +405,35 @@ function scrapeWork(doc, url) {
 		translator.translate();
 	}, null, null, {
 		Referer: 'https://trove.nla.gov.au/',
-		apikey: '3b84ac7cec64f3e346f9a8f063230949'
-		// the API key is the one that the site uses for client-side requests,
-		// so it and the Referer have to be exactly right.
+		apikey: apiKeyGen(doc || docContext)
 	});
 }
 
+
+// Get a cookie's value by key from the document.
+function getCookie(doc, key) {
+	let field = doc.cookie.split("; ").find(row => row.startsWith(`${key}=`));
+	return field ? field.split("=")[1] : undefined;
+}
+
+
+// Compute the API key using cookie info.
+// See the source under Webpack path trove-vue/src/service/services.js
+function apiKeyGen(doc) {
+	let xctx = getCookie(doc, "x-ctx");
+	if (typeof xctx === "undefined") {
+		return "";
+	}
+	return md5("Wonder" + xctx).replace(/^0+/, "");
+}
+
+
+// Minfied MD5 digest function.
+// See https://pajhome.org.uk/crypt/md5/md5.html
+/* eslint-disable */
+function md5(d){function rstr2hex(d){for(var _,m="0123456789abcdef",f="",r=0;r>>4&15)+m.charAt(15&_);return f}function rstr2binl(d){for(var _=Array(d.length>>2),m=0;m<_.length;m++)_[m]=0;for(m=0;m<8*d.length;m+=8)_[m>>5]|=(255&d.charCodeAt(m/8))<>5]>>>m%32&255);return _}function binl_md5(d,_){d[_>>5]|=128<<_%32,d[14+(_+64>>>9<<4)]=_;for(var m=1732584193,f=-271733879,r=-1732584194,i=271733878,n=0;n>16)+(_>>16)+(m>>16)<<16|65535&m}function bit_rol(d,_){return d<<_|d>>>32-_}return rstr2hex(binl2rstr(binl_md5(rstr2binl(d),8*d.length)))}
+/* eslint-enable */
+
 /** BEGIN TEST CASES **/
 var testCases = [
 	{

From 6bae3cf344e3213879fda975ab5db7c13093e4c4 Mon Sep 17 00:00:00 2001
From: Dan Stillman 
Date: Mon, 12 Jun 2023 01:59:49 -0400
Subject: [PATCH 033/125] Update `ecmaVersion` to 2018

Spread operator, async iteration, and some other things that were
supported in Firefox 60 (where these still have to run until Zotero 7)
---
 .eslintrc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.eslintrc b/.eslintrc
index 4369f8576a7..3ad2bebfd66 100644
--- a/.eslintrc
+++ b/.eslintrc
@@ -7,7 +7,7 @@
 		"@zotero"
 	],
 	"parserOptions": {
-		"ecmaVersion": 2017
+		"ecmaVersion": 2018
 	},
 	"globals": {
 		"Zotero": "readonly",

From e9a8ceacc26ddec57aff4930f1c5cb5c7a135939 Mon Sep 17 00:00:00 2001
From: Abe Jellinek 
Date: Mon, 12 Jun 2023 10:52:55 -0400
Subject: [PATCH 034/125] Update Web of Science Nextgen SID extraction

Fixes #3050
---
 Web of Science Nextgen.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Web of Science Nextgen.js b/Web of Science Nextgen.js
index e1e828e3bd2..dffe5e1140a 100644
--- a/Web of Science Nextgen.js	
+++ b/Web of Science Nextgen.js	
@@ -9,7 +9,7 @@
 	"inRepository": true,
 	"translatorType": 4,
 	"browserSupport": "gcsibv",
-	"lastUpdated": "2022-04-22 20:47:13"
+	"lastUpdated": "2023-06-12 14:52:18"
 }
 
 /*
@@ -156,7 +156,7 @@ function getItemID(url) {
 }
 
 function getSessionID(doc, callback) {
-	const sidRegex = /sid=([a-zA-Z0-9]+)/i;
+	const sidRegex = /(?:sid=|"SID":")([a-zA-Z0-9]+)/i;
 	
 	// session ID is embedded in the static page inside an inline