diff --git a/plugins/image/service/src/processor.spec.ts b/plugins/image/service/src/processor.spec.ts index 9457e179b..b38ecbcc1 100644 --- a/plugins/image/service/src/processor.spec.ts +++ b/plugins/image/service/src/processor.spec.ts @@ -1,4 +1,4 @@ -import { EventProcessingState, FindUnprocessedImageAttachments, ImageContent, ImageDescriptor, ImagePluginConfig, ImageService, createImagePluginControl, orientAttachmentImage, processImageAttachments, thumbnailAttachmentImage, UnprocessedAttachmentReference, ImagePluginControl, defaultImagePluginConfig } from './processor' +import { EventProcessingState, FindUnprocessedImageAttachments, ImageContent, ImageDescriptor, ImagePluginConfig, ImageService, createImagePluginControl, orientAttachmentImage, processImageAttachments, thumbnailAttachmentImage, UnprocessedAttachmentReference, ImagePluginControl, defaultImagePluginConfig, AttachmentProcessingResult } from './processor' import { MageEventRepository, MageEventAttrs, MageEventId, MageEvent, copyMageEventAttrs } from '@ngageoint/mage.service/lib/entities/events/entities.events' import { Attachment, AttachmentContentPatchAttrs, AttachmentId, AttachmentPatchAttrs, AttachmentStore, AttachmentStoreError, AttachmentStoreErrorCode, copyObservationAttrs, copyThumbnailAttrs, EventScopedObservationRepository, FormEntry, Observation, ObservationAttrs, ObservationId, patchAttachment, StagedAttachmentContent, StagedAttachmentContentId, StagedAttachmentContentRef, Thumbnail, ThumbnailContentPatchAttrs } from '@ngageoint/mage.service/lib/entities/observations/entities.observations' import stream from 'stream' @@ -7,6 +7,7 @@ import { BufferWriteable } from './util.spec' import { FormFieldType } from '@ngageoint/mage.service/lib/entities/events/entities.events.forms' import _ from 'lodash' import { PluginStateRepository } from '@ngageoint/mage.service/lib/plugins.api' +import { FindUnprocessedAttachments } from './adapters.db.mongo' function minutes(x: number): number { return 1000 * 60 * x @@ -68,7 +69,7 @@ function sameAsObservationWithoutDates(expected: Observation): jasmine.Asymmetri const { createdAt, lastModified, ...expectedAttrs } = copyObservationAttrs(expected) expectedAttrs.attachments.forEach(x => delete x.lastModified) return { - asymmetricMatch(actual: any, matchersUtil: jasmine.MatchersUtil) { + asymmetricMatch(actual: any, matchersUtil: jasmine.MatchersUtil): boolean { const { createdAt, lastModified, ...actualAttrs } = copyObservationAttrs(actual) actualAttrs.attachments.forEach(x => delete x.lastModified) return actual instanceof Observation && matchersUtil.equals(expectedAttrs, actualAttrs) @@ -116,7 +117,7 @@ function observationWithAttachments(id: ObservationId, event: MageEvent, attachm const asyncIterableOf = (items: T[]): AsyncIterable => { return { - async *[Symbol.asyncIterator]() { + async *[Symbol.asyncIterator](): AsyncGenerator { for (const item of items) { yield Promise.resolve(item) } @@ -126,10 +127,10 @@ const asyncIterableOf = (items: T[]): AsyncIterable => { function closeTo(target: number, delta: number): jasmine.AsymmetricMatcher { return { - asymmetricMatch(other) { + asymmetricMatch(other): boolean { return Math.abs(target - other) <= delta }, - jasmineToString(prettyPrint) { + jasmineToString(prettyPrint): string { return `a number within ${delta} of ${target}` } } @@ -138,7 +139,7 @@ function closeTo(target: number, delta: number): jasmine.AsymmetricMatcher closeTo(Date.now(), 100) +const closeToNow = (): jasmine.AsymmetricMatcher => closeTo(Date.now(), 100) describe('processing interval', () => { @@ -175,7 +176,7 @@ describe('processing interval', () => { describe('orient phase', () => { - it('orients the attachment image and patches the attachment', async () => { + it('orients the attachment image and produces an attachment patch', async () => { const att: Attachment = Object.freeze({ id: '1.123.1', @@ -192,10 +193,7 @@ describe('processing interval', () => { const originalConstentStream = stream.Readable.from(originalContent) const stagedContent = new StagedAttachmentContent('stage1', new BufferWriteable()) const obsBefore: Observation = observationWithAttachments('1.123', event1, [ att ]) - const obsStored: Observation = patchAttachment(obsBefore, att.id, { size: 321321, width: 1000, height: 1200, oriented: true }) as Observation - const attStored = obsStored.attachmentFor(att.id) as Attachment const obsRepo = observationRepos.get(event1.id)! - obsRepo.patchAttachment.and.resolveTo(obsStored) attachmentStore.readContent.and.resolveTo(originalConstentStream) attachmentStore.stagePendingContent.and.resolveTo(stagedContent) attachmentStore.saveContent.and.resolveTo({ size: 321321, contentLocator: att.contentLocator! }) @@ -209,22 +207,18 @@ describe('processing interval', () => { dimensions: { width: 1000, height: 1200 }, } }) - const oriented = await orientAttachmentImage(obsBefore, att.id, imageService, obsRepo, attachmentStore, console) as Observation + const oriented = await orientAttachmentImage(obsBefore, att.id, imageService, attachmentStore, console) const orientedContent = stagedContent.tempLocation as BufferWriteable - expect(oriented).toBeInstanceOf(Observation) - expect(oriented.attachments).toEqual([ - { ...att, lastModified: attStored.lastModified, size: 321321, width: 1000, height: 1200, oriented: true, contentLocator: att.contentLocator! } - ]) + expect(oriented.patch).toEqual({ contentType: 'image/jpeg', size: 321321, width: 1000, height: 1200, oriented: true, contentLocator: att.contentLocator! }) expect(orientedContent.content).toEqual(Buffer.from('photo of penguins')) - expect(obsRepo.patchAttachment).toHaveBeenCalledOnceWith( - obsBefore, att.id, { oriented: true, size: 321321, width: 1000, height: 1200, contentType: 'image/jpeg', contentLocator: att.contentLocator! }) expect(imageService.autoOrient).toHaveBeenCalledOnceWith(jasmine.objectContaining({ bytes: originalConstentStream }), stagedContent.tempLocation) expect(attachmentStore.saveContent).toHaveBeenCalledOnceWith(stagedContent, att.id, obsBefore) + expect(obsRepo.patchAttachment).not.toHaveBeenCalled() expect(obsRepo.save).not.toHaveBeenCalled() }) - it('marks the attachment oriented if content does not exist', async () => { + it('does not produce an attachment patch if the content does not exist', async () => { const att: Attachment = Object.freeze({ id: '1.123.1', @@ -238,25 +232,18 @@ describe('processing interval', () => { contentLocator: String(Date.now()) }) const obsBefore: Observation = observationWithAttachments('1.123', event1, [ att ]) - const obsStored: Observation = patchAttachment(obsBefore, att.id, { oriented: true }) as Observation - const attStored = obsStored.attachmentFor(att.id) as Attachment const obsRepo = observationRepos.get(event1.id)! - obsRepo.patchAttachment.and.resolveTo(obsStored) attachmentStore.readContent.and.resolveTo(null) - const oriented = await orientAttachmentImage(obsBefore, att.id, imageService, obsRepo, attachmentStore, console) as Observation + const oriented = await orientAttachmentImage(obsBefore, att.id, imageService, attachmentStore, console) - expect(oriented).toBeInstanceOf(Observation) - expect(oriented.attachments).toEqual([ - { ...att, oriented: true, lastModified: attStored.lastModified, width: undefined, height: undefined } - ]) - expect(obsRepo.patchAttachment).toHaveBeenCalledOnceWith( - obsBefore, att.id, { oriented: true }) + expect(oriented.patch).toBeUndefined() expect(imageService.autoOrient).not.toHaveBeenCalled() expect(attachmentStore.saveContent).not.toHaveBeenCalled() + expect(obsRepo.patchAttachment).not.toHaveBeenCalled() expect(obsRepo.save).not.toHaveBeenCalled() }) - it('marks the attachment oriented if reading content fails', async () => { + it('does not produce an attachment patch if reading content fails', async () => { const att: Attachment = Object.freeze({ id: '1.123.1', @@ -270,21 +257,48 @@ describe('processing interval', () => { contentLocator: String(Date.now()) }) const obsBefore: Observation = observationWithAttachments('1.123', event1, [ att ]) - const obsStored: Observation = patchAttachment(obsBefore, att.id, { oriented: true }) as Observation - const attStored = obsStored.attachmentFor(att.id) as Attachment const obsRepo = observationRepos.get(event1.id)! - obsRepo.patchAttachment.and.resolveTo(obsStored) attachmentStore.readContent.and.resolveTo(new AttachmentStoreError(AttachmentStoreErrorCode.ContentNotFound)) - const oriented = await orientAttachmentImage(obsBefore, att.id, imageService, obsRepo, attachmentStore, console) as Observation + const oriented = await orientAttachmentImage(obsBefore, att.id, imageService, attachmentStore, console) - expect(oriented).toBeInstanceOf(Observation) - expect(oriented.attachments).toEqual([ - { ...att, oriented: true, lastModified: attStored.lastModified, width: undefined, height: undefined } - ]) - expect(obsRepo.patchAttachment).toHaveBeenCalledOnceWith( - obsBefore, att.id, { oriented: true }) + expect(oriented.patch).toBeUndefined() expect(imageService.autoOrient).not.toHaveBeenCalled() expect(attachmentStore.saveContent).not.toHaveBeenCalled() + expect(obsRepo.patchAttachment).not.toHaveBeenCalled() + expect(obsRepo.save).not.toHaveBeenCalled() + }) + + it('does not produce an attachment patch if the image service cannot decode the image', async () => { + + const att: Attachment = Object.freeze({ + id: '1.123.1', + observationFormId: 'form1', + fieldName: 'field1', + oriented: false, + thumbnails: [], + contentType: 'image/jpeg', + name: 'test1.jpeg', + size: 320000, + contentLocator: String(Date.now()) + }) + const obsBefore: Observation = observationWithAttachments('1.123', event1, [ att ]) + const obsRepo = observationRepos.get(event1.id)! + attachmentStore.readContent.and.resolveTo(stream.Readable.from(Buffer.from('corrupted image'))) + const staged: StagedAttachmentContent = { + id: 'nawgonwork', + tempLocation: jasmine.createSpyObj( + 'mockPendingContent', + [ 'write', 'end' ] + ) + } + attachmentStore.stagePendingContent.and.resolveTo(staged) + imageService.autoOrient.and.resolveTo(new Error('wut is this')) + const oriented = await orientAttachmentImage(obsBefore, att.id, imageService, attachmentStore, console) + + expect(oriented.patch).toBeUndefined() + expect(attachmentStore.saveContent).not.toHaveBeenCalled() + expect(staged.tempLocation.write).not.toHaveBeenCalled() + expect(obsRepo.patchAttachment).not.toHaveBeenCalled() expect(obsRepo.save).not.toHaveBeenCalled() }) @@ -294,7 +308,7 @@ describe('processing interval', () => { describe('thumbnail phase', () => { - it('generates the specified thumbnails and patches the attachment once', async () => { + it('generates the specified thumbnails and produces the attachment patch', async () => { const att: Attachment = Object.freeze({ id: '1.456.1', @@ -314,7 +328,7 @@ describe('processing interval', () => { constructor(readonly metaData: Thumbnail, stagedContentId: StagedAttachmentContentId) { this.stagedContent = new StagedAttachmentContent(stagedContentId, new BufferWriteable()) } - get stagedContentBytes() { return (this.stagedContent.tempLocation as BufferWriteable).content } + get stagedContentBytes(): Buffer { return (this.stagedContent.tempLocation as BufferWriteable).content } } const salt = String(Date.now()) const expectedThumbs = new Map() @@ -349,26 +363,12 @@ describe('processing interval', () => { sizeInBytes: minDimension * 100 } }) - obsRepo.patchAttachment.and.callFake(async (obs, attId, patch) => { - return patchAttachment(obs, attId, patch) as Observation - }) - const obsActual = await thumbnailAttachmentImage(obsBefore, att.id, [ 60, 120, 240 ], imageService, obsRepo, attachmentStore, console) as Observation - - expect(copyObservationAttrs(obsActual)).toEqual( - { - ...copyObservationAttrs(obsBefore), - lastModified: jasmine.any(Date), - attachments: [ - { ...att, lastModified: jasmine.any(Date), thumbnails: Array.from(expectedThumbs.values()).map(x => x.metaData) } - ] - } - ) + + const thumbResult = await thumbnailAttachmentImage(obsBefore, att.id, [ 60, 120, 240 ], imageService, attachmentStore, console) + expect(thumbResult.patch).toEqual({ thumbnails: Array.from(expectedThumbs.values()).map(x => x.metaData) }) expect(expectedThumbs.get(60)?.stagedContentBytes.toString()).toEqual('big majestic mountains @60') expect(expectedThumbs.get(120)?.stagedContentBytes.toString()).toEqual('big majestic mountains @120') expect(expectedThumbs.get(240)?.stagedContentBytes.toString()).toEqual('big majestic mountains @240') - expect(obsRepo.patchAttachment).toHaveBeenCalledOnceWith(obsBefore, att.id, { - thumbnails: Array.from(expectedThumbs.values()).map(x => x.metaData) - }) expect(attachmentStore.stagePendingContent).toHaveBeenCalledTimes(3) expect(attachmentStore.readContent).toHaveBeenCalledTimes(3) expect(attachmentStore.readContent.calls.argsFor(0)).toEqual([ att.id, obsBefore ]) @@ -383,6 +383,7 @@ describe('processing interval', () => { expect(attachmentStore.saveThumbnailContent).toHaveBeenCalledWith(expectedThumbs.get(120)?.stagedContent, 120, att.id, sameAsObservationWithoutDates(obsStaged)) expect(attachmentStore.saveThumbnailContent).toHaveBeenCalledWith(expectedThumbs.get(240)?.stagedContent, 240, att.id, sameAsObservationWithoutDates(obsStaged)) expect(attachmentStore.saveContent).not.toHaveBeenCalled() + expect(obsRepo.patchAttachment).not.toHaveBeenCalled() expect(obsRepo.save).not.toHaveBeenCalled() }) @@ -791,7 +792,7 @@ describe('processing interval', () => { processed.processed = true return processed }, - async *[Symbol.asyncIterator]() { + async *[Symbol.asyncIterator](): AsyncGenerator { while (this.cursor < unprocessedAttachments.length - 1) { if (this.cursor < 0 || this.current?.processed === true) { this.cursor += 1 @@ -836,6 +837,77 @@ describe('processing interval', () => { } }) + it('marks attachment oriented when orient phase does not produce a patch', async () => { + + const eventProcessingStates = new Map([ + { event: copyMageEventAttrs(event1), latestAttachmentProcessedTimestamp: Date.now() - 1000 * 60 * 5 }, + { event: copyMageEventAttrs(event2), latestAttachmentProcessedTimestamp: Date.now() - 1000 * 60 * 2 }, + ].map(x => [ x.event.id, x ])) + const pluginState: ImagePluginConfig = { + enabled: true, + intervalSeconds: 60, + intervalBatchSize: 1000, + thumbnailSizes: [] + } + const attachment1: Attachment = { + id: '1.100.1', + fieldName: 'field1', + lastModified: new Date(minutesAgo(6)), + observationFormId: 'form1', + oriented: false, + thumbnails: [], + } + const attachment2: Attachment = { + id: '2.200.1', + fieldName: 'field1', + lastModified: new Date(), + observationFormId: 'form1', + oriented: false, + thumbnails: [] + } + const obs1 = observationWithAttachments('1.100', event1, [ attachment1 ]) + const obs2 = observationWithAttachments('2.200', event2, [ attachment2 ]) + const unprocessedAttachments: UnprocessedAttachmentReference[] = [ + { eventId: obs1.eventId, observationId: obs1.id, attachmentId: attachment1.id }, + { eventId: obs2.eventId, observationId: obs2.id, attachmentId: attachment2.id } + ] + const findUnprocessedAttachments = jasmine.createSpy('findAttachments') + .and.resolveTo(asyncIterableOf(unprocessedAttachments)) + const invalidImageBytes = stream.Readable.from(Buffer.from('wut is this')) + const validImageBytes = stream.Readable.from(Buffer.from('goats.png')) + observationRepos.get(event1.id)?.findById.withArgs(obs1.id).and.resolveTo(obs1) + observationRepos.get(event2.id)?.findById.withArgs(obs2.id).and.resolveTo(obs2) + observationRepos.get(event1.id)?.patchAttachment.and.resolveTo(obs1) + observationRepos.get(event2.id)?.patchAttachment.and.resolveTo(obs2) + attachmentStore.readContent.withArgs(jasmine.stringMatching(attachment1.id), jasmine.anything()).and.resolveTo(invalidImageBytes) + attachmentStore.readContent.withArgs(jasmine.stringMatching(attachment2.id), jasmine.anything()).and.resolveTo(validImageBytes) + attachmentStore.stagePendingContent.and.resolveTo({ + tempLocation: new BufferWriteable(), + id: 'pending' + }) + imageService.autoOrient.and.callFake(async (source: ImageContent, dest: NodeJS.WritableStream) => { + if (source.bytes === invalidImageBytes) { + return new Error('bad image data') + } + return await Promise.resolve>({ + mediaType: `image/png`, + dimensions: { width: 100, height: 120 }, + sizeInBytes: 10000 + }) + }) + await processImageAttachments(pluginState, eventProcessingStates, + findUnprocessedAttachments, imageService, eventRepo, observationRepoForEvent, attachmentStore, console) + + expect(observationRepos.get(event1.id)?.patchAttachment).toHaveBeenCalledOnceWith(obs1, attachment1.id, { oriented: true }) + expect(observationRepos.get(event2.id)?.patchAttachment).toHaveBeenCalledOnceWith(obs2, attachment2.id, { + oriented: true, + contentType: 'image/png', + size: 10000, + width: 100, + height: 120, + }) + }) + it('does not generate thumbnails if orient fails') it('updates the attachment after orienting and creating thumbnails') @@ -939,7 +1011,7 @@ class BufferAttachmentStore implements AttachmentStore { * @param count * @returns */ -function someRunLoops(count: number = 100): Promise { +function someRunLoops(count = 100): Promise { return new Promise(function waitRemaining(resolve: () => any, reject: (err: any) => any, remaining: number = count) { if (remaining === 0) { return resolve() diff --git a/plugins/image/service/src/processor.ts b/plugins/image/service/src/processor.ts index 82b74e79f..cab357369 100644 --- a/plugins/image/service/src/processor.ts +++ b/plugins/image/service/src/processor.ts @@ -1,5 +1,5 @@ import { MageEventAttrs, MageEventId, MageEventRepository } from '@ngageoint/mage.service/lib/entities/events/entities.events' -import { Attachment, AttachmentId, AttachmentStore, Observation, ObservationId, ObservationRepositoryForEvent, AttachmentPatchAttrs, Thumbnail, EventScopedObservationRepository, AttachmentStoreError, putAttachmentThumbnailForMinDimension, StagedAttachmentContent } from '@ngageoint/mage.service/lib/entities/observations/entities.observations' +import { Attachment, AttachmentId, AttachmentStore, Observation, ObservationId, ObservationRepositoryForEvent, AttachmentPatchAttrs, Thumbnail, EventScopedObservationRepository, AttachmentStoreError, putAttachmentThumbnailForMinDimension, StagedAttachmentContent, patchAttachment } from '@ngageoint/mage.service/lib/entities/observations/entities.observations' import { PluginStateRepository } from '@ngageoint/mage.service/lib/plugins.api' import path from 'path' @@ -148,6 +148,24 @@ export type EventProcessingState = { latestAttachmentProcessedTimestamp: number } +/** + TODO: + reads zero-byte file (?) and fails - mark as processed + +2023-09-27T09:31:50.738Z - [mage.image] error processing attachment { +eventId: 17, +observationId: '619ec65d7dc44d090239b266', +attachmentId: '619ec6677dc44d090239b268' +} +-- process result: [Error: pngload_buffer: libspng read error vips2png: unable to write to target target] + +2023-09-27T20:26:57.400Z - [mage.image] error processing attachment { + eventId: 17, + observationId: '619fbe0b7dc44d090239b4d4', + attachmentId: '619fbe107dc44d090239b4d6' +} +-- process result: [Error: Input buffer contains unsupported image format] + */ export async function processImageAttachments( pluginState: ImagePluginConfig, eventProcessingStates: Map | null, @@ -164,30 +182,42 @@ export async function processImageAttachments( eventProcessingStates = syncProcessingStatesFromAllEvents(allEvents, eventProcessingStates) const eventLatestModifiedTimes = new Map() const unprocessedAttachments = await findUnprocessedAttachments(Array.from(eventProcessingStates.values()), null, startTime, pluginState.intervalBatchSize) + unprocessedAttachments[Symbol.asyncIterator] let processedCount = 0 for await (const unprocessed of unprocessedAttachments) { // TODO: check results for errors console.info(`processing attachment`, unprocessed) const { observationId, attachmentId } = unprocessed const observationRepo = await observationRepoForEvent(unprocessed.eventId) - const orient = async (observation: Observation) => orientAttachmentImage(observation, attachmentId, imageService, observationRepo, attachmentStore, console) - const thumbnail = async (observation: Observation) => thumbnailAttachmentImage(observation, attachmentId, pluginState.thumbnailSizes, imageService, observationRepo, attachmentStore, console) + const orient = async (observation: Observation): Promise => + orientAttachmentImage(observation, attachmentId, imageService, attachmentStore, console) + const thumbnail = async (observation: Observation): Promise => + thumbnailAttachmentImage(observation, attachmentId, pluginState.thumbnailSizes, imageService, attachmentStore, console) const [ original, processed ] = await observationRepo.findById(observationId) - .then(checkObservationThen(orient)) - .then(checkObservationThen(thumbnail)) - if (original instanceof Observation && processed instanceof Observation) { - const eventLatestModified = eventLatestModifiedTimes.get(unprocessed.eventId) || 0 - const attachment = original.attachmentFor(attachmentId) - const attachmentLastModified = attachment?.lastModified?.getTime() || original.lastModified.getTime() - if (attachmentLastModified > eventLatestModified) { - eventLatestModifiedTimes.set(unprocessed.eventId, attachmentLastModified) + .then(saveResultOf(orient, observationRepo)) + .then(saveResultOf(thumbnail, observationRepo)) + if (original instanceof Observation) { + if (processed instanceof Observation) { + const eventLatestModified = eventLatestModifiedTimes.get(unprocessed.eventId) || 0 + const attachment = original.attachmentFor(attachmentId) + const attachmentLastModified = attachment?.lastModified?.getTime() || original.lastModified.getTime() + if (attachmentLastModified > eventLatestModified) { + eventLatestModifiedTimes.set(unprocessed.eventId, attachmentLastModified) + } + console.info(`processed attachment ${attachment?.name || ''}`, unprocessed) } - console.info(`processed attachment ${attachment?.name || ''}`, unprocessed) - } - else { - console.error(`error processing attachment`, unprocessed, '\n-- process result:', processed) + else { + console.error(`error processing attachment`, unprocessed, '\n-- process result:', processed) + const attachment = original.attachmentFor(attachmentId) + if (attachment && !attachment.oriented) { + const oriented = await observationRepo.patchAttachment(original, attachmentId, { oriented: true }) + if (oriented instanceof Error) { + console.error(`error marking attachment oriented after failed processing:`, unprocessed, oriented) + } + } + } + processedCount++ } - processedCount++ } console.info(`finished image attachment processing interval - ${processedCount} attachments`) return new Map(Array.from(eventProcessingStates.entries(), ([ eventId, state ]) => { @@ -195,39 +225,44 @@ export async function processImageAttachments( })) } +export class AttachmentProcessingResult { + + constructor( + readonly observation: Observation, + readonly attachmentId: AttachmentId, + readonly patch?: AttachmentPatchAttrs, + readonly error?: Error, + ) {} + + get success(): boolean { return !this.error } +} + export async function orientAttachmentImage ( observation: Observation, attachmentId: AttachmentId, imageService: ImageService, - observationRepo: EventScopedObservationRepository, attachmentStore: AttachmentStore, console: Console -): Promise { +): Promise { const attachment = observation.attachmentFor(attachmentId) if (!attachment) { - return new Error(`attachment ${attachmentId} does not exist on observation ${observation.id}`) + return new AttachmentProcessingResult(observation, attachmentId, undefined, Error(`attachment ${attachmentId} does not exist on observation ${observation.id}`)) } const content = await attachmentStore.readContent(attachmentId, observation) if (!content || content instanceof Error) { console.error(`error reading content of image attachment ${attachmentId} observation ${observation.id}:`, content || 'content not found') - const updatedObservation = await observationRepo.patchAttachment(observation, attachmentId, { oriented: true }) - if (updatedObservation instanceof Observation) { - return updatedObservation - } - const errMsg = `error marking attachment ${attachmentId} oriented on observation ${observation.id} after reading content failed:` - const errReason = updatedObservation ? String(updatedObservation) : 'observation not found' - console.error(errMsg, updatedObservation || errReason) - return new Error(`${errMsg} ${errReason}`) + return new AttachmentProcessingResult(observation, attachmentId, undefined, content || new Error('content not found')) } const pending = await attachmentStore.stagePendingContent() const oriented = await imageService.autoOrient(imageContentForAttachment(attachment, content), pending.tempLocation) if (oriented instanceof Error) { - return oriented + console.error(`error orienting attachment ${attachmentId} on observation ${observation.id} at ${attachment.contentLocator}`, oriented) + return new AttachmentProcessingResult(observation, attachmentId, undefined, oriented) } const storeResult = await attachmentStore.saveContent(pending, attachment.id, observation) if (storeResult instanceof AttachmentStoreError) { console.error(`error saving pending oriented content ${pending.id} for attachment ${attachmentId} on observation ${observation.id}:`, storeResult) - return storeResult + return new AttachmentProcessingResult(observation, attachmentId, undefined, storeResult) } const patch: AttachmentPatchAttrs = { oriented: true, @@ -236,28 +271,19 @@ export async function orientAttachmentImage ( ...oriented.dimensions, ...storeResult, } - const updatedObservation = await observationRepo.patchAttachment(observation, attachmentId, patch) - if (!updatedObservation) { - const err = new Error(`observation ${observation.id} did not exist after orienting attachment ${attachmentId}`) - console.error(err) - return err - } - else if (updatedObservation instanceof Error) { - console.error(`error updating oriented attachment ${attachment.id} on observation ${observation.id}:`, updatedObservation) - return updatedObservation - } - return updatedObservation + return new AttachmentProcessingResult(observation, attachmentId, patch) } export async function thumbnailAttachmentImage( observation: Observation, attachmentId: AttachmentId, thumbnailSizes: number[], - imageService: ImageService, observationRepo: EventScopedObservationRepository, attachmentStore: AttachmentStore, - console: Console): Promise { + imageService: ImageService, attachmentStore: AttachmentStore, console: Console): Promise { + if (thumbnailSizes.length === 0) { + return new AttachmentProcessingResult(observation, attachmentId) + } const attachment = observation.attachmentFor(attachmentId) if (!attachment) { const err = new Error(`attachment ${attachmentId} does not exist on observation ${observation.id}`) - console.error(err) - return err + return new AttachmentProcessingResult(observation, attachmentId, undefined, err) } /* TODO: this thumbnail meta-data and content saving sequence is pretty awkward, @@ -285,13 +311,7 @@ export async function thumbnailAttachmentImage( return putAttachmentThumbnailForMinDimension(obsWithThumbs, attachmentId, storedThumb) as Observation }, obsWithThumbs) const storedThumbPatch: AttachmentPatchAttrs = { thumbnails: obsWithThumbs.attachmentFor(attachmentId)!.thumbnails } - const savedObsWithThumbs = await observationRepo.patchAttachment(observation, attachmentId, storedThumbPatch) - if (!(savedObsWithThumbs instanceof Observation)) { - if (!savedObsWithThumbs) { - return new Error(`observation ${observation.id} did not exist after thumbnail operation on attachment ${attachment.id}`) - } - } - return savedObsWithThumbs + return new AttachmentProcessingResult(observation, attachmentId, storedThumbPatch) } function syncProcessingStatesFromAllEvents(allEvents: MageEventAttrs[], states: Map | null | undefined): Map { @@ -304,14 +324,24 @@ function syncProcessingStatesFromAllEvents(allEvents: MageEventAttrs[], states: return newStates } -type ObservationUpdateResult = [ original: Observation | Error | null, updated: Observation | Error | null ] -function checkObservationThen(update: (o: Observation) => Promise): (target: Observation | Error | null | ObservationUpdateResult) => Promise { - return async target => { - const [ original, updated ] = Array.isArray(target) ? target : [ target, target ] - if (updated instanceof Observation) { - return await update(updated).then(result => [ original, result ]) +type ObservationUpdateResult = [ Observation | Error | null, Observation | Error | null ] + +/** + * Perform the given attachment processing operation. If the operation + * produces an attachment patch, apply the patch and save the observation. + */ +function saveResultOf(processAttachment: (o: Observation) => Promise, repo: EventScopedObservationRepository): (target: Observation | null | Error | ObservationUpdateResult) => Promise { + return async (target): Promise => { + const [ original, next ] = Array.isArray(target) ? target : [ target, target ] + if (original instanceof Observation && next instanceof Observation) { + const result = await processAttachment(next) + if (result.patch) { + const patched = await repo.patchAttachment(next, result.attachmentId, result.patch) + return [ original, result.error || patched ] + } + return [ original, result.error || original ] } - return [ original, updated ] + return [ original, next ] } }