Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add links to version & asset metadata to web view #76

Merged
merged 6 commits into from
Feb 15, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ In Development
- Log errors that cause 404 and 500 responses
- Added breadcrumbs to HTML views of collections
- `FAST_NOT_EXIST` components are now checked for case-insensitively
- Add links to version & asset metadata to the web view

v0.2.0 (2024-02-07)
-------------------
Expand Down
88 changes: 57 additions & 31 deletions src/dandi/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -111,12 +111,22 @@
pub(crate) fn get_all_dandisets(
&self,
) -> impl Stream<Item = Result<Dandiset, DandiError>> + '_ {
self.paginate(self.get_url(["dandisets"]))
self.paginate::<RawDandiset>(self.get_url(["dandisets"]))
.map_ok(|ds| ds.with_metadata_urls(self))

Check warning on line 115 in src/dandi/mod.rs

View check run for this annotation

Codecov / codecov/patch

src/dandi/mod.rs#L114-L115

Added lines #L114 - L115 were not covered by tests
}

pub(crate) fn dandiset(&self, dandiset_id: DandisetId) -> DandisetEndpoint<'_> {
DandisetEndpoint::new(self, dandiset_id)
}

fn version_metadata_url(&self, dandiset_id: &DandisetId, version_id: &VersionId) -> Url {
self.get_url([
"dandisets",
dandiset_id.as_ref(),
"versions",
version_id.as_ref(),
])
}

Check warning on line 129 in src/dandi/mod.rs

View check run for this annotation

Codecov / codecov/patch

src/dandi/mod.rs#L122-L129

Added lines #L122 - L129 were not covered by tests
}

#[derive(Clone, Debug)]
Expand All @@ -139,21 +149,29 @@

pub(crate) async fn get(&self) -> Result<Dandiset, DandiError> {
self.client
.get(
.get::<RawDandiset>(

Check warning on line 152 in src/dandi/mod.rs

View check run for this annotation

Codecov / codecov/patch

src/dandi/mod.rs#L152

Added line #L152 was not covered by tests
self.client
.get_url(["dandisets", self.dandiset_id.as_ref()]),
)
.await
.map(|ds| ds.with_metadata_urls(self.client))

Check warning on line 157 in src/dandi/mod.rs

View check run for this annotation

Codecov / codecov/patch

src/dandi/mod.rs#L157

Added line #L157 was not covered by tests
}

pub(crate) fn get_all_versions(
&self,
) -> impl Stream<Item = Result<DandisetVersion, DandiError>> + '_ {
self.client.paginate(self.client.get_url([
"dandisets",
self.dandiset_id.as_ref(),
"versions",
]))
self.client
.paginate::<RawDandisetVersion>(self.client.get_url([
"dandisets",
self.dandiset_id.as_ref(),
"versions",
]))
.map_ok(|v| {
let url = self
.client
.version_metadata_url(&self.dandiset_id, &v.version);
v.with_metadata_url(url)
})

Check warning on line 174 in src/dandi/mod.rs

View check run for this annotation

Codecov / codecov/patch

src/dandi/mod.rs#L163-L174

Added lines #L163 - L174 were not covered by tests
}
}

Expand All @@ -175,32 +193,45 @@

pub(crate) async fn get(&self) -> Result<DandisetVersion, DandiError> {
self.client
.get(self.client.get_url([
.get::<RawDandisetVersion>(self.client.get_url([

Check warning on line 196 in src/dandi/mod.rs

View check run for this annotation

Codecov / codecov/patch

src/dandi/mod.rs#L196

Added line #L196 was not covered by tests
"dandisets",
self.dandiset_id.as_ref(),
"versions",
self.version_id.as_ref(),
"info",
]))
.await
.map(|v| v.with_metadata_url(self.metadata_url()))
}

Check warning on line 205 in src/dandi/mod.rs

View check run for this annotation

Codecov / codecov/patch

src/dandi/mod.rs#L204-L205

Added lines #L204 - L205 were not covered by tests

fn metadata_url(&self) -> Url {
self.client
.version_metadata_url(&self.dandiset_id, &self.version_id)
}

Check warning on line 210 in src/dandi/mod.rs

View check run for this annotation

Codecov / codecov/patch

src/dandi/mod.rs#L207-L210

Added lines #L207 - L210 were not covered by tests

fn asset_metadata_url(&self, asset_id: &str) -> Url {
self.client.get_url([
"dandisets",
self.dandiset_id.as_ref(),
"versions",
self.version_id.as_ref(),
"assets",
asset_id,
])

Check warning on line 220 in src/dandi/mod.rs

View check run for this annotation

Codecov / codecov/patch

src/dandi/mod.rs#L212-L220

Added lines #L212 - L220 were not covered by tests
}

pub(crate) async fn get_metadata(&self) -> Result<VersionMetadata, DandiError> {
let data = self
.client
.get::<serde_json::Value>(self.client.get_url([
"dandisets",
self.dandiset_id.as_ref(),
"versions",
self.version_id.as_ref(),
]))
.get::<serde_json::Value>(self.metadata_url())

Check warning on line 226 in src/dandi/mod.rs

View check run for this annotation

Codecov / codecov/patch

src/dandi/mod.rs#L226

Added line #L226 was not covered by tests
.await?;
Ok(VersionMetadata(dump_json_as_yaml(data).into_bytes()))
}

async fn get_asset_by_id(&self, id: &str) -> Result<Asset, DandiError> {
self.client
.get(self.client.get_url([
let raw_asset = self
.client
.get::<RawAsset>(self.client.get_url([

Check warning on line 234 in src/dandi/mod.rs

View check run for this annotation

Codecov / codecov/patch

src/dandi/mod.rs#L232-L234

Added lines #L232 - L234 were not covered by tests
"dandisets",
self.dandiset_id.as_ref(),
"versions",
Expand All @@ -209,7 +240,8 @@
id,
"info",
]))
.await
.await?;
raw_asset.try_into_asset(self).map_err(Into::into)

Check warning on line 244 in src/dandi/mod.rs

View check run for this annotation

Codecov / codecov/patch

src/dandi/mod.rs#L243-L244

Added lines #L243 - L244 were not covered by tests
}

pub(crate) fn get_root_children(
Expand Down Expand Up @@ -272,14 +304,14 @@
.append_pair("metadata", "1")
.append_pair("order", "path");
let dirpath = path.to_dir_path();
let stream = self.client.paginate::<Asset>(url.clone());
let stream = self.client.paginate::<RawAsset>(url.clone());

Check warning on line 307 in src/dandi/mod.rs

View check run for this annotation

Codecov / codecov/patch

src/dandi/mod.rs#L307

Added line #L307 was not covered by tests
tokio::pin!(stream);
while let Some(asset) = stream.try_next().await? {
if asset.path() == path {
return Ok(AtAssetPath::Asset(asset));
} else if asset.path().is_strictly_under(&dirpath) {
if &asset.path == path {
return Ok(AtAssetPath::Asset(asset.try_into_asset(self)?));
} else if asset.path.is_strictly_under(&dirpath) {

Check warning on line 312 in src/dandi/mod.rs

View check run for this annotation

Codecov / codecov/patch

src/dandi/mod.rs#L310-L312

Added lines #L310 - L312 were not covered by tests
return Ok(AtAssetPath::Folder(AssetFolder { path: dirpath }));
} else if asset.path().as_ref() > dirpath.as_ref() {
} else if asset.path.as_ref() > dirpath.as_ref() {

Check warning on line 314 in src/dandi/mod.rs

View check run for this annotation

Codecov / codecov/patch

src/dandi/mod.rs#L314

Added line #L314 was not covered by tests
break;
}
}
Expand Down Expand Up @@ -308,18 +340,10 @@
match self.get_path(&zarr_path).await? {
AtAssetPath::Folder(_) => continue,
AtAssetPath::Asset(Asset::Blob(_)) => {
let mut url = self.client.get_url([
"dandisets",
self.dandiset_id.as_ref(),
"versions",
self.version_id.as_ref(),
"assets",
]);
url.query_pairs_mut().append_pair("path", path.as_ref());
return Err(DandiError::PathUnderBlob {
path: path.clone(),
blob_path: zarr_path,
});
})

Check warning on line 346 in src/dandi/mod.rs

View check run for this annotation

Codecov / codecov/patch

src/dandi/mod.rs#L346

Added line #L346 was not covered by tests
}
AtAssetPath::Asset(Asset::Zarr(zarr)) => {
let s3 = self.client.get_s3client_for_zarr(&zarr).await?;
Expand Down Expand Up @@ -414,6 +438,8 @@
source: ZarrToS3Error,
},
#[error(transparent)]
AssetType(#[from] AssetTypeError),
#[error(transparent)]
S3(#[from] S3Error),
}

Expand Down
126 changes: 86 additions & 40 deletions src/dandi/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,29 +13,78 @@
}

#[derive(Clone, Debug, Deserialize, Eq, PartialEq)]
pub(crate) struct Dandiset {
pub(crate) identifier: DandisetId,
pub(crate) struct RawDandiset {
identifier: DandisetId,
#[serde(with = "time::serde::rfc3339")]
pub(crate) created: OffsetDateTime,
created: OffsetDateTime,
#[serde(with = "time::serde::rfc3339")]
pub(crate) modified: OffsetDateTime,
modified: OffsetDateTime,
//contact_person: String,
//embargo_status: ...,
draft_version: RawDandisetVersion,
most_recent_published_version: Option<RawDandisetVersion>,
}

impl RawDandiset {
pub(super) fn with_metadata_urls(self, client: &super::DandiClient) -> Dandiset {
let draft_version = self
.draft_version
.with_metadata_url(client.version_metadata_url(&self.identifier, &VersionId::Draft));
let most_recent_published_version = self.most_recent_published_version.map(|v| {
let url = client.version_metadata_url(&self.identifier, &v.version);
v.with_metadata_url(url)
});
Dandiset {
identifier: self.identifier,
created: self.created,
modified: self.modified,
draft_version,
most_recent_published_version,
}
}

Check warning on line 44 in src/dandi/types.rs

View check run for this annotation

Codecov / codecov/patch

src/dandi/types.rs#L29-L44

Added lines #L29 - L44 were not covered by tests
}

#[derive(Clone, Debug, Eq, PartialEq)]

Check warning on line 47 in src/dandi/types.rs

View check run for this annotation

Codecov / codecov/patch

src/dandi/types.rs#L47

Added line #L47 was not covered by tests
pub(crate) struct Dandiset {
pub(crate) identifier: DandisetId,
pub(crate) created: OffsetDateTime,
pub(crate) modified: OffsetDateTime,
pub(crate) draft_version: DandisetVersion,
pub(crate) most_recent_published_version: Option<DandisetVersion>,
}

#[derive(Clone, Debug, Deserialize, Eq, PartialEq)]
pub(crate) struct DandisetVersion {
pub(crate) version: VersionId,
pub(crate) struct RawDandisetVersion {
pub(super) version: VersionId,
//name: String,
//asset_count: u64,
pub(crate) size: i64,
size: i64,
//status: ...,
#[serde(with = "time::serde::rfc3339")]
pub(crate) created: OffsetDateTime,
created: OffsetDateTime,
#[serde(with = "time::serde::rfc3339")]
modified: OffsetDateTime,
}

impl RawDandisetVersion {
pub(super) fn with_metadata_url(self, metadata_url: Url) -> DandisetVersion {
DandisetVersion {
version: self.version,
size: self.size,
created: self.created,
modified: self.modified,
metadata_url,
}
}

Check warning on line 78 in src/dandi/types.rs

View check run for this annotation

Codecov / codecov/patch

src/dandi/types.rs#L70-L78

Added lines #L70 - L78 were not covered by tests
}

#[derive(Clone, Debug, Eq, PartialEq)]

Check warning on line 81 in src/dandi/types.rs

View check run for this annotation

Codecov / codecov/patch

src/dandi/types.rs#L81

Added line #L81 was not covered by tests
pub(crate) struct DandisetVersion {
pub(crate) version: VersionId,
pub(crate) size: i64,
pub(crate) created: OffsetDateTime,
pub(crate) modified: OffsetDateTime,
pub(crate) metadata_url: Url,
}

#[derive(Clone, Debug, Eq, PartialEq)]
Expand Down Expand Up @@ -95,28 +144,19 @@
asset_id: String,
}

#[allow(clippy::large_enum_variant)]
#[derive(Clone, Debug, Eq, PartialEq)]
pub(crate) enum AtAssetPath {
Folder(AssetFolder),
Asset(Asset),
}

#[derive(Clone, Debug, Deserialize, Eq, PartialEq)]
#[serde(try_from = "RawAsset")]
#[derive(Clone, Debug, Eq, PartialEq)]

Check warning on line 154 in src/dandi/types.rs

View check run for this annotation

Codecov / codecov/patch

src/dandi/types.rs#L154

Added line #L154 was not covered by tests
pub(crate) enum Asset {
Blob(BlobAsset),
Zarr(ZarrAsset),
}

impl Asset {
pub(crate) fn path(&self) -> &PurePath {
match self {
Asset::Blob(a) => &a.path,
Asset::Zarr(a) => &a.path,
}
}
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub(crate) struct BlobAsset {
pub(crate) asset_id: String,
Expand All @@ -126,6 +166,7 @@
pub(crate) created: OffsetDateTime,
pub(crate) modified: OffsetDateTime,
pub(crate) metadata: AssetMetadata,
pub(crate) metadata_url: Url,
}

impl BlobAsset {
Expand Down Expand Up @@ -160,6 +201,7 @@
pub(crate) created: OffsetDateTime,
pub(crate) modified: OffsetDateTime,
pub(crate) metadata: AssetMetadata,
pub(crate) metadata_url: Url,
}

impl ZarrAsset {
Expand Down Expand Up @@ -211,11 +253,11 @@
}

#[derive(Clone, Debug, Deserialize, Eq, PartialEq)]
struct RawAsset {
pub(super) struct RawAsset {
asset_id: String,
blob: Option<String>,
zarr: Option<String>,
path: PurePath,
pub(super) path: PurePath,
size: i64,
#[serde(with = "time::serde::rfc3339")]
created: OffsetDateTime,
Expand All @@ -224,34 +266,38 @@
metadata: AssetMetadata,
}

impl TryFrom<RawAsset> for Asset {
type Error = AssetTypeError;

fn try_from(value: RawAsset) -> Result<Asset, AssetTypeError> {
match (value.blob, value.zarr) {
impl RawAsset {
pub(super) fn try_into_asset(
self,
endpoint: &super::VersionEndpoint<'_>,
) -> Result<Asset, AssetTypeError> {
let metadata_url = endpoint.asset_metadata_url(&self.asset_id);
match (self.blob, self.zarr) {

Check warning on line 275 in src/dandi/types.rs

View check run for this annotation

Codecov / codecov/patch

src/dandi/types.rs#L270-L275

Added lines #L270 - L275 were not covered by tests
(Some(blob_id), None) => Ok(Asset::Blob(BlobAsset {
asset_id: value.asset_id,
asset_id: self.asset_id,

Check warning on line 277 in src/dandi/types.rs

View check run for this annotation

Codecov / codecov/patch

src/dandi/types.rs#L277

Added line #L277 was not covered by tests
blob_id,
path: value.path,
size: value.size,
created: value.created,
modified: value.modified,
metadata: value.metadata,
path: self.path,
size: self.size,
created: self.created,
modified: self.modified,
metadata: self.metadata,
metadata_url,

Check warning on line 284 in src/dandi/types.rs

View check run for this annotation

Codecov / codecov/patch

src/dandi/types.rs#L279-L284

Added lines #L279 - L284 were not covered by tests
})),
(None, Some(zarr_id)) => Ok(Asset::Zarr(ZarrAsset {
asset_id: value.asset_id,
asset_id: self.asset_id,

Check warning on line 287 in src/dandi/types.rs

View check run for this annotation

Codecov / codecov/patch

src/dandi/types.rs#L287

Added line #L287 was not covered by tests
zarr_id,
path: value.path,
size: value.size,
created: value.created,
modified: value.modified,
metadata: value.metadata,
path: self.path,
size: self.size,
created: self.created,
modified: self.modified,
metadata: self.metadata,
metadata_url,

Check warning on line 294 in src/dandi/types.rs

View check run for this annotation

Codecov / codecov/patch

src/dandi/types.rs#L289-L294

Added lines #L289 - L294 were not covered by tests
})),
(None, None) => Err(AssetTypeError::Neither {
asset_id: value.asset_id,
asset_id: self.asset_id,

Check warning on line 297 in src/dandi/types.rs

View check run for this annotation

Codecov / codecov/patch

src/dandi/types.rs#L297

Added line #L297 was not covered by tests
}),
(Some(_), Some(_)) => Err(AssetTypeError::Both {
asset_id: value.asset_id,
asset_id: self.asset_id,

Check warning on line 300 in src/dandi/types.rs

View check run for this annotation

Codecov / codecov/patch

src/dandi/types.rs#L300

Added line #L300 was not covered by tests
}),
}
}
Expand Down
Loading