Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Restructure database file hierarchy #240

Merged
merged 4 commits into from
Feb 17, 2024
Merged

Restructure database file hierarchy #240

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
43ada62
Restrict triggers to tei.xml files and disable deletion trigger
cmil Feb 5, 2024
4446437
Restructure file hierarchy
cmil Feb 3, 2024
2e6eb07
Fix load:load-corpus
cmil Feb 5, 2024
f3c720f
Fix webhook processing
cmil Feb 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion api.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1084,7 +1084,7 @@ paths:
example: |
{
"message": "Delivery accepted.",
"result": "/db/data/dracor/webhook/20190121213010-xxx.xml",
"result": "/db/dracor/webhook/20190121213010-xxx.xml",
"scheduled": true
}

Expand Down
49 changes: 34 additions & 15 deletions jobs/process-webhook-delivery.xq
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ declare function local:update (
let $collection := substring(
$target, 1, string-length($target) - string-length($filename) - 1
)
let $l := util:log("info", "Fetching " || $source)
let $l := util:log-system-out("fetching " || $source)
let $response := local:gh-request("get", $source)
let $status := string($response[1]/@status)

Expand All @@ -60,17 +60,19 @@ declare function local:update (
(: FIXME: make sure $data is valid TEI :)

return if ($data) then
if (
util:log("info", "Updating " || $target),
xmldb:store(
$collection,
xmldb:encode-uri($filename),
$data
)
) then true() else false()
try {
util:log-system-out("webhook update: " || $target),
xmldb:create-collection("/", $collection) and
xmldb:store($collection, xmldb:encode-uri($filename), $data)
} catch * {
util:log-system-out($err:description),
false()
}
else false()
else (
util:log("warn", "Fetching " || $source || " failed. Status: " || $status),
util:log-system-out(
"Failed to fetch " || $source || "; status: " || $status
),
false()
)
};
Expand All @@ -82,9 +84,12 @@ declare function local:remove ($file as xs:string) as xs:boolean {
)
return
try {
(xmldb:remove($collection, $filename), true())
if ($filename = "tei.xml") then
(xmldb:remove($collection), true())
else
(xmldb:remove($collection, $filename), true())
} catch * {
util:log("warn", $err:description), false()
util:log-system-out($err:description), false()
}
};

Expand All @@ -93,6 +98,18 @@ declare function local:make-url ($template, $path) {
return $url
};

declare function local:make-target ($path, $corpusname) {
$config:corpora-root || '/' || $corpusname || '/' || (
if ($path = "corpus.xml")
then $path
else replace(
replace($path, '^' || $config:corpus-repo-prefix || '/', ''),
"\.xml$",
"/tei.xml"
)
)
};

declare function local:get-repo-contents ($url-template) {
let $url := local:make-url($url-template, $config:corpus-repo-prefix)
let $response := local:gh-request("get", $url)
Expand All @@ -104,7 +121,7 @@ declare function local:process-delivery () {
let $delivery := collection($config:webhook-root)
/delivery[@id = $local:delivery and not(@processed)]
let $repo := $delivery/@repo/string()
let $corpus := collection($config:data-root)//tei:teiCorpus[
let $corpus := collection($config:corpora-root)//tei:teiCorpus[
tei:teiHeader//tei:publicationStmt/tei:idno[@type="repo" and . = $repo]
]

Expand All @@ -127,9 +144,11 @@ declare function local:process-delivery () {
let $source := if ($path = "corpus.xml")
then local:make-url($contents-url, $path)
else $contents?*[?type = "file" and ?path = $path]?git_url
let $target := $config:data-root || '/' || $corpusname || '/'
|| replace($path, '^' || $config:corpus-repo-prefix || '/', '')
let $target := local:make-target($path, $corpusname)
let $action := $file/@action
let $log := util:log-system-out(
"action: "||$action||" target: "||$target||" source: "||$source
)

let $result := if ($action = "remove") then
local:remove($target)
Expand Down
136 changes: 33 additions & 103 deletions modules/api.xqm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,23 +166,6 @@ declare function local:get-index-keys ($collection as xs:string, $elem as xs:str
</terms>
};

declare function local:id-to-url ($id, $accept) {
let $tei := collection($config:data-root)/tei:TEI[@xml:id = $id][1]
let $parts := tokenize(base-uri($tei), "[/.]")
let $corpusname := $parts[last()-2]
let $playname := $parts[last()-1]

return if ($tei) then
if ($accept = "application/rdf+xml") then
$config:api-base || "/corpora/" || $corpusname || "/plays/" || $playname || "/rdf"
else if ($accept = "application/json") then
$config:api-base || "/corpora/" || $corpusname || "/plays/" || $playname
else
let $p := tokenize($config:api-base, '/')
return $p[1] || '//' || $p[3] || '/' || $corpusname || "/" || $playname
else ()
};

(:~
: Resolve DraCor ID of a play
:
Expand All @@ -199,7 +182,7 @@ declare
%rest:path("/v1/id/{$id}")
%rest:header-param("Accept", "{$accept}")
function api:id-to-url($id, $accept) {
let $url := local:id-to-url($id, $accept)
let $url := dutil:id-to-url($id, $accept)
return if (not($url)) then
<rest:response>
<http:response status="404"/>
Expand All @@ -213,10 +196,8 @@ function api:id-to-url($id, $accept) {
};

declare function local:get-corpus-metrics ($corpus as xs:string) {
let $collection-uri := concat($config:data-root, "/", $corpus)
let $collection-uri := concat($config:corpora-root, "/", $corpus)
let $col := collection($collection-uri)
let $metrics-uri := concat($config:metrics-root, "/", $corpus)
let $metrics := collection($metrics-uri)
let $num-plays := count($col/tei:TEI)
let $list := $col//tei:particDesc/tei:listPerson
let $num-characters := count($list/(tei:person|tei:personGrp))
Expand All @@ -234,11 +215,11 @@ declare function local:get-corpus-metrics ($corpus as xs:string) {
"sp": $num-sp,
"stage": $num-stage,
"wordcount": map {
"text": sum($metrics//text),
"sp": sum($metrics//sp),
"stage": sum($metrics//stage)
"text": sum($col/metrics/text),
"sp": sum($col/metrics/sp),
"stage": sum($col/metrics/stage)
},
"updated": max($metrics//metrics/xs:dateTime(@updated))
"updated": max($col/metrics/xs:dateTime(@updated))
}
};

Expand All @@ -256,7 +237,7 @@ declare
%output:method("json")
function api:corpora($include) {
array {
for $corpus in collection($config:data-root)//tei:teiCorpus
for $corpus in collection($config:corpora-root)//tei:teiCorpus
let $info := dutil:get-corpus-info($corpus)
let $name := $info?name
order by $name
Expand Down Expand Up @@ -344,20 +325,11 @@ function api:corpora-post-tei($data, $auth) {
"error": "corpus already exists"
}
) else (
let $tei-dir := concat($config:data-root, '/', $name)
return (
util:log-system-out("creating corpus"),
util:log-system-out($data),
xmldb:create-collection($config:data-root, $name),
xmldb:create-collection($config:metrics-root, $name),
xmldb:create-collection($config:sitelinks-root, $name),
xmldb:create-collection($config:rdf-root, $name),
xmldb:store($tei-dir, "corpus.xml", $data),
map {
"name": $name,
"title": $title
}
)
dutil:create-corpus($name, $data/tei:teiCorpus),
map {
"name": $name,
"title": $title
}
)
};

Expand Down Expand Up @@ -411,46 +383,10 @@ function api:corpora-post-json($data) {
"message": "Only lower case ASCII letters and digits are accepted."
}
)
else
let $corpus :=
<teiCorpus xmlns="http://www.tei-c.org/ns/1.0">
<teiHeader>
<fileDesc>
<titleStmt>
<title>{$json?title}</title>
</titleStmt>
<publicationStmt>
<idno type="URI" xml:base="https://dracor.org/">{$name}</idno>
{
if ($json?repository)
then <idno type="repo">{$json?repository}</idno>
else ()
}
</publicationStmt>
</fileDesc>
{if ($json?description) then (
<encodingDesc>
<projectDesc>
{
for $p in tokenize($json?description, "&#10;&#10;")
return <p>{$p}</p>
}
</projectDesc>
</encodingDesc>
) else ()}
</teiHeader>
</teiCorpus>
let $tei-dir := concat($config:data-root, '/', $name)
return (
util:log-system-out("creating corpus"),
util:log-system-out($corpus),
xmldb:create-collection($config:data-root, $name),
xmldb:create-collection($config:metrics-root, $name),
xmldb:create-collection($config:sitelinks-root, $name),
xmldb:create-collection($config:rdf-root, $name),
xmldb:store($tei-dir, "corpus.xml", $corpus),
$json
)
else (
dutil:create-corpus($json),
$json
)
};

(:~
Expand All @@ -472,7 +408,7 @@ function api:corpus-index($corpusname) {
let $corpus := dutil:get-corpus-info-by-name($corpusname)
let $title := $corpus?title
let $description := $corpus?description
let $collection := concat($config:data-root, "/", $corpusname)
let $collection := $config:corpora-root || "/" || $corpusname
let $col := collection($collection)
return
if (not($corpus?name) or not(xmldb:collection-available($collection))) then
Expand All @@ -483,17 +419,16 @@ function api:corpus-index($corpusname) {
$corpus,
map:entry("plays", array {
for $tei in $col//tei:TEI
let $filename := tokenize(base-uri($tei), "/")[last()]
let $name := tokenize($filename, "\.")[1]
let $paths := dutil:filepaths(base-uri($tei))
let $name := $paths?playname
let $id := dutil:get-dracor-id($tei)
let $titles := dutil:get-titles($tei)
let $titlesEng := dutil:get-titles($tei, 'eng')
let $years := dutil:get-years-iso($tei)
let $authors := dutil:get-authors($tei)
let $play-uri :=
$config:api-base || "/corpora/" || $corpusname || "/plays/" || $name
let $metrics-url :=
$config:metrics-root || "/" || $corpusname || "/" || $filename
let $metrics-url := $paths?files?metrics
let $network-size := doc($metrics-url)//network/size/text()
let $yearNormalized := dutil:get-normalized-year($tei)
let $premiere-date := dutil:get-premiere-date($tei)
Expand Down Expand Up @@ -645,14 +580,11 @@ function api:delete-corpus($corpusname, $auth) {
<http:response status="404"/>
</rest:response>
else
let $url := $config:data-root || "/" || $corpusname || "/corpus.xml"
let $url := $config:corpora-root || "/" || $corpusname || "/corpus.xml"
return
if ($url = $corpus/base-uri()) then
(
xmldb:remove($config:data-root || "/" || $corpusname),
xmldb:remove($config:metrics-root || "/" || $corpusname),
xmldb:remove($config:sitelinks-root || "/" || $corpusname),
xmldb:remove($config:rdf-root || "/" || $corpusname),
xmldb:remove($config:corpora-root || "/" || $corpusname),
map {
"message": "corpus deleted",
"uri": $url
Expand Down Expand Up @@ -748,7 +680,7 @@ declare
%rest:path("/v1/corpora/{$corpusname}/word-frequencies/{$elem}")
%rest:produces("application/xml", "text/xml")
function api:word-frequencies-xml($corpusname, $elem) {
let $collection := concat($config:data-root, "/", $corpusname)
let $collection := concat($config:corpora-root, "/", $corpusname)
let $terms := local:get-index-keys($collection, $elem)
return $terms
};
Expand All @@ -760,7 +692,7 @@ declare
%output:media-type("text/csv")
%output:method("text")
function api:word-frequencies-csv($corpusname, $elem) {
let $collection := concat($config:data-root, "/", $corpusname)
let $collection := concat($config:corpora-root, "/", $corpusname)
let $terms := local:get-index-keys($collection, $elem)
for $t in $terms/term
order by number($t/@count) descending
Expand Down Expand Up @@ -811,17 +743,15 @@ function api:play-delete($corpusname, $playname, $data, $auth) {
</rest:response>
else

let $doc := dutil:get-doc($corpusname, $playname)
let $paths := dutil:filepaths($corpusname, $playname)

return
if (not($doc)) then
if (not(doc($paths?files?tei))) then
<rest:response>
<http:response status="404"/>
</rest:response>
else
let $filename := $playname || ".xml"
let $collection := $config:data-root || "/" || $corpusname
return (xmldb:remove($collection, $filename))
xmldb:remove($paths?collections?play)
};

(:~
Expand Down Expand Up @@ -937,9 +867,10 @@ function api:play-tei-put($corpusname, $playname, $data, $auth) {
<message>TEI document required</message>
)
else
let $filename := $playname || ".xml"
let $collection := $config:data-root || "/" || $corpusname
let $result := xmldb:store($collection, $filename, $data/tei:TEI)
let $collection := xmldb:create-collection(
$config:corpora-root || "/" || $corpusname, $playname
)
let $result := xmldb:store($collection, "tei.xml", $data/tei:TEI)
return $data
};

Expand All @@ -955,9 +886,8 @@ declare
%rest:produces("application/xml", "text/xml")
%output:media-type("application/xml")
function api:play-rdf($corpusname, $playname) {
let $url := $config:rdf-root || "/" || $corpusname || "/" || $playname
|| ".rdf.xml"
let $doc := doc($url)
let $paths := dutil:filepaths($corpusname, $playname)
let $doc := doc($paths?files?rdf)
return
if (not($doc)) then
<rest:response>
Expand Down
18 changes: 6 additions & 12 deletions modules/config.xqm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,12 @@ declare variable $config:app-root :=
substring-before($modulePath, "/modules")
;

declare variable $config:file := "/db/data/dracor/config-v1.xml";
declare variable $config:secrets-file := "/db/data/dracor/secrets.xml";
declare variable $config:dracor-root := "/db/dracor";
declare variable $config:corpora-root := $config:dracor-root || "/corpora";
declare variable $config:webhook-root := $config:dracor-root || "/webhook";

declare variable $config:file := $config:dracor-root || "/config-v1.xml";
declare variable $config:secrets-file := $config:dracor-root || "/secrets.xml";

(:
The base URL under which the REST API is hosted.
Expand All @@ -43,16 +47,6 @@ declare variable $config:secrets-file := "/db/data/dracor/secrets.xml";
declare variable $config:api-base :=
doc($config:file)//api-base/normalize-space();

declare variable $config:data-root := "/db/data/dracor/tei";

declare variable $config:rdf-root := "/db/data/dracor/rdf";

declare variable $config:metrics-root := "/db/data/dracor/metrics";

declare variable $config:sitelinks-root := "/db/data/dracor/sitelinks";

declare variable $config:webhook-root := "/db/data/dracor/webhook";

declare variable $config:webhook-secret :=
doc($config:secrets-file)//gh-webhook/normalize-space();

Expand Down
Loading
Loading