diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index 83589a83e..4c8829563 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -469,9 +469,15 @@ interface Api { deleteScoringResult?(datasetId: string, resultId: string): Promise; /** Annotation sets, revisions or on-disk files a source on this dataset can point at. */ listScoringSources?(datasetId: string): Promise; - /** Datasets that may be named as the other side of a comparison. */ + /** + * Datasets that may be named as the other side of a comparison; also the + * dataset list the review page offers. + */ listScoringDatasets?(): Promise; - /** Open a platform dataset picker; returns null when the user cancels. */ + /** + * Open a platform dataset picker; returns null when the user cancels. + * Shared by the scoring and review pages. + */ pickScoringDataset?(excludeIds: string[]): Promise; /** Save a text export where the user chooses; resolves false when they cancel. */ saveScoringExport?(args: { filename: string; mime: string; content: string }): Promise; @@ -485,6 +491,12 @@ interface Api { ): Promise; loadConfig(datasetId: string): Promise; + /** + * loadConfig without the platform's viewer bookkeeping (desktop recents, + * web browse location), for pages that read many datasets at once such as + * Review. Callers fall back to loadConfig when absent. + */ + peekConfig?(datasetId: string): Promise; loadDetections(datasetId: string, revision?: number, set?: string): Promise; loadFrameMetadata(datasetId: string): Promise; diff --git a/client/dive-common/components/DatasetPicker.vue b/client/dive-common/components/DatasetPicker.vue new file mode 100644 index 000000000..c8d160fa0 --- /dev/null +++ b/client/dive-common/components/DatasetPicker.vue @@ -0,0 +1,265 @@ + + + + + diff --git a/client/dive-common/components/Review/ReviewCell.vue b/client/dive-common/components/Review/ReviewCell.vue new file mode 100644 index 000000000..d2f41e3b7 --- /dev/null +++ b/client/dive-common/components/Review/ReviewCell.vue @@ -0,0 +1,401 @@ + + + + + diff --git a/client/dive-common/components/Review/ReviewChip.vue b/client/dive-common/components/Review/ReviewChip.vue new file mode 100644 index 000000000..2849cafc5 --- /dev/null +++ b/client/dive-common/components/Review/ReviewChip.vue @@ -0,0 +1,1225 @@ + + + + + diff --git a/client/dive-common/components/Review/ReviewDatasetsPanel.vue b/client/dive-common/components/Review/ReviewDatasetsPanel.vue new file mode 100644 index 000000000..6e3fa67ee --- /dev/null +++ b/client/dive-common/components/Review/ReviewDatasetsPanel.vue @@ -0,0 +1,200 @@ + + + + + diff --git a/client/dive-common/components/Review/ReviewGrid.vue b/client/dive-common/components/Review/ReviewGrid.vue new file mode 100644 index 000000000..153b1fa27 --- /dev/null +++ b/client/dive-common/components/Review/ReviewGrid.vue @@ -0,0 +1,81 @@ + + + + + diff --git a/client/dive-common/components/Review/ReviewGridControls.vue b/client/dive-common/components/Review/ReviewGridControls.vue new file mode 100644 index 000000000..29d47de15 --- /dev/null +++ b/client/dive-common/components/Review/ReviewGridControls.vue @@ -0,0 +1,157 @@ + + + + + diff --git a/client/dive-common/components/Review/ReviewPage.vue b/client/dive-common/components/Review/ReviewPage.vue new file mode 100644 index 000000000..16602f819 --- /dev/null +++ b/client/dive-common/components/Review/ReviewPage.vue @@ -0,0 +1,800 @@ + + + + + diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index e5710ec55..dc7dba566 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -175,6 +175,16 @@ export default defineComponent({ type: Boolean, default: false, }, + /** Deep link: frame to seek to once the media is ready (e.g. from the review grid). */ + initialFrame: { + type: Number as PropType, + default: undefined, + }, + /** Deep link: track to select once annotations are loaded. */ + initialTrackId: { + type: Number as PropType, + default: undefined, + }, }, setup(props, { emit }) { const { prompt, visible } = usePrompt(); @@ -2033,6 +2043,32 @@ export default defineComponent({ }; loadData(); + /** + * Apply a deep link (initialFrame / initialTrackId) once: after the + * annotations are loaded and the media controller reports its frame + * range, so the seek is not swallowed by the annotator's own init seek. + */ + let initialFocusApplied = false; + watch( + () => [progress.loaded, aggregateController.value.maxFrame.value] as const, + ([loaded, maxFrame]) => { + if (initialFocusApplied || !loaded) return; + if (props.initialFrame === undefined && props.initialTrackId === undefined) return; + if (props.initialFrame !== undefined && maxFrame <= 0) return; + initialFocusApplied = true; + nextTick(() => { + if (props.initialFrame !== undefined) { + handler.seekFrame(Math.min(props.initialFrame, maxFrame)); + } + if (props.initialTrackId !== undefined + && cameraStore.getAnyPossibleTrack(props.initialTrackId)) { + handler.trackSelect(props.initialTrackId, false); + } + }); + }, + { immediate: true }, + ); + const reloadAnnotations = async () => { progress.loaded = false; discardChanges(); diff --git a/client/dive-common/datasetPicker.spec.ts b/client/dive-common/datasetPicker.spec.ts new file mode 100644 index 000000000..33fbcf00c --- /dev/null +++ b/client/dive-common/datasetPicker.spec.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from 'vitest'; +import { filterDatasetRows, selectableIds } from './datasetPicker'; + +const rows = [ + { + id: 'a', name: 'Amchitka East', type: 'image-sequence', fps: 5, + }, + { + id: 'b', name: 'Bering clip', type: 'video', fps: 30, + }, + { id: 'c', name: 'Caton rig', type: 'multi' }, +]; + +describe('filterDatasetRows', () => { + it('matches any listed field, ignoring case and surrounding space', () => { + expect(filterDatasetRows(rows, ' VIDEO ').map((r) => r.id)).toEqual(['b']); + expect(filterDatasetRows(rows, 'ri').map((r) => r.id)).toEqual(['b', 'c']); + expect(filterDatasetRows(rows, '30', ['fps']).map((r) => r.id)).toEqual(['b']); + expect(filterDatasetRows(rows, '')).toHaveLength(3); + }); +}); + +describe('selectableIds', () => { + it('leaves out what is already selected', () => { + expect(selectableIds(rows, ['b'])).toEqual(['a', 'c']); + expect(selectableIds(filterDatasetRows(rows, 'rig'), ['c'])).toEqual([]); + }); +}); diff --git a/client/dive-common/datasetPicker.ts b/client/dive-common/datasetPicker.ts new file mode 100644 index 000000000..c4bf35723 --- /dev/null +++ b/client/dive-common/datasetPicker.ts @@ -0,0 +1,37 @@ +/** + * Rows and filtering behind the shared dataset picker, kept free of Vue so + * the search behaviour is testable and identical on every page. + */ + +/** A dataset offered for selection; any extra fields can feed extra table columns. */ +export interface DatasetPickerRow { + id: string; + name: string; + type?: string; +} + +/** + * Rows whose listed fields contain the search text (case-insensitive, + * whitespace-trimmed). An empty search keeps everything. + */ +export function filterDatasetRows( + rows: readonly T[], + search: string, + fields: readonly string[] = ['name', 'type'], +): T[] { + const needle = search.trim().toLowerCase(); + if (!needle) return [...rows]; + return rows.filter((row) => fields.some((field) => { + const value = (row as unknown as Record)[field]; + return value !== undefined && value !== null && String(value).toLowerCase().includes(needle); + })); +} + +/** Ids of the listed rows not yet selected: what "select all" adds. */ +export function selectableIds( + rows: readonly T[], + selectedIds: readonly string[], +): string[] { + const selected = new Set(selectedIds); + return rows.filter((row) => !selected.has(row.id)).map((row) => row.id); +} diff --git a/client/dive-common/review/chipRenderer.spec.ts b/client/dive-common/review/chipRenderer.spec.ts new file mode 100644 index 000000000..1903cf9f2 --- /dev/null +++ b/client/dive-common/review/chipRenderer.spec.ts @@ -0,0 +1,66 @@ +import { + chipRegion, chipScale, chipSizeFor, toChipPoint, toImagePoint, +} from './chipRenderer'; + +describe('chipRegion', () => { + it('is a square centred on the box, padded on every side', () => { + const region = chipRegion([10, 20, 30, 60], 0.5); + // Longer side 40, padded by 50% each side -> 80. + expect(region).toEqual({ + x: 20 - 40, y: 40 - 40, width: 80, height: 80, + }); + }); + + it('extends the square to the requested aspect ratio, keeping the box centred', () => { + const wide = chipRegion([10, 20, 30, 60], 0, 2); + expect(wide).toEqual({ + x: 20 - 40, y: 40 - 20, width: 80, height: 40, + }); + const tall = chipRegion([10, 20, 30, 60], 0, 0.5); + expect(tall).toEqual({ + x: 20 - 20, y: 40 - 40, width: 40, height: 80, + }); + expect(chipRegion([0, 0, 10, 10], 0, Number.NaN).width).toBe(10); + }); + + it('tolerates inverted or degenerate boxes', () => { + expect(chipRegion([30, 60, 10, 20], 0).width).toBe(40); + expect(chipRegion([5, 5, 5, 5], 0).width).toBe(1); + expect(chipRegion([0, 0, 10, 10], -1).width).toBe(10); + }); +}); + +describe('chipSizeFor', () => { + it('rounds cell sizes up to a bucket and caps at the largest', () => { + expect(chipSizeFor(100)).toBe(128); + expect(chipSizeFor(128)).toBe(128); + expect(chipSizeFor(129)).toBe(192); + expect(chipSizeFor(5000)).toBe(768); + }); +}); + +describe('chipScale', () => { + it('fills the requested size, upscaling small crops and downscaling large ones', () => { + expect(chipScale({ + x: 0, y: 0, width: 32, height: 32, + }, 256)).toBe(8); + expect(chipScale({ + x: 0, y: 0, width: 1024, height: 512, + }, 256)).toBe(0.25); + }); +}); + +describe('chip point mapping', () => { + it('round-trips between image and chip coordinates', () => { + const transform = { + region: { + x: 100, y: 50, width: 40, height: 20, + }, + scale: 4, + width: 160, + height: 80, + }; + expect(toChipPoint(transform, 110, 60)).toEqual([40, 40]); + expect(toImagePoint(transform, 40, 40)).toEqual([110, 60]); + }); +}); diff --git a/client/dive-common/review/chipRenderer.ts b/client/dive-common/review/chipRenderer.ts new file mode 100644 index 000000000..33ed44207 --- /dev/null +++ b/client/dive-common/review/chipRenderer.ts @@ -0,0 +1,145 @@ +/** + * Crops one box out of a decoded frame into a chip image. The crop is a + * square centred on the box so the object stays centred as a track cycles + * through frames of different sizes; area past the frame edge is left dark + * rather than shifting the object off centre. + */ +import type { RectBounds } from 'vue-media-annotator/utils'; +import type { DecodedFrame } from './frameSource'; + +export interface ChipRenderOptions { + /** Context around the box as a fraction of its longer side. */ + padding: number; + /** Longer output edge in pixels. */ + size: number; + /** Output width / height; the crop region widens or heightens to match. */ + aspect?: number; + /** Box outline colour; omit to draw no outline. */ + outline?: string; + /** JPEG quality. */ + quality?: number; +} + +export interface ChipRegion { + x: number; + y: number; + width: number; + height: number; +} + +/** How a rendered chip maps to the frame: chip px = (image px - region.x) * scale. */ +export interface ChipTransform { + region: ChipRegion; + scale: number; + /** Chip image size in pixels. */ + width: number; + height: number; +} + +export interface RenderedChip { + dataUrl: string; + transform: ChipTransform; +} + +/** Chip pixel coordinates of an image point. */ +export function toChipPoint(transform: ChipTransform, x: number, y: number): [number, number] { + return [(x - transform.region.x) * transform.scale, (y - transform.region.y) * transform.scale]; +} + +/** Image coordinates of a chip pixel. */ +export function toImagePoint(transform: ChipTransform, x: number, y: number): [number, number] { + return [transform.region.x + x / transform.scale, transform.region.y + y / transform.scale]; +} + +/** + * Crop region (may extend past the image) for a box with padding: the + * padded square around the box, widened or heightened to the requested + * aspect ratio so the object stays centred whatever the cell's shape. + */ +export function chipRegion(bounds: RectBounds, padding: number, aspect = 1): ChipRegion { + const [x1, y1, x2, y2] = bounds; + const w = Math.max(1, Math.abs(x2 - x1)); + const h = Math.max(1, Math.abs(y2 - y1)); + const side = Math.max(w, h) * (1 + 2 * Math.max(0, padding)); + const ratio = aspect > 0 && Number.isFinite(aspect) ? aspect : 1; + const width = ratio >= 1 ? side * ratio : side; + const height = ratio >= 1 ? side : side / ratio; + const cx = (x1 + x2) / 2; + const cy = (y1 + y2) / 2; + return { + x: cx - width / 2, y: cy - height / 2, width, height, + }; +} + +/** Pixel sizes chips are rendered at; cells pick the smallest that covers them. */ +export const CHIP_SIZE_BUCKETS = [128, 192, 256, 384, 512, 768]; + +export function chipSizeFor(cellPixels: number): number { + const wanted = Math.ceil(cellPixels); + return CHIP_SIZE_BUCKETS.find((size) => size >= wanted) ?? CHIP_SIZE_BUCKETS[CHIP_SIZE_BUCKETS.length - 1]; +} + +/** + * Scale from frame pixels to chip pixels: the crop's longer side fills the + * requested size. Small crops are upscaled so the chip is rendered once at + * the cell's resolution with high-quality resampling, instead of the browser + * stretching a tiny image (and its compression artifacts) on every paint. + */ +export function chipScale(region: ChipRegion, size: number): number { + const longest = Math.max(1, Math.max(region.width, region.height)); + return Math.max(16, size) / longest; +} + +export function renderChip(frame: DecodedFrame, bounds: RectBounds, options: ChipRenderOptions): RenderedChip { + const region = chipRegion(bounds, options.padding, options.aspect); + const scale = chipScale(region, options.size); + const canvas = document.createElement('canvas'); + canvas.width = Math.max(1, Math.round(region.width * scale)); + canvas.height = Math.max(1, Math.round(region.height * scale)); + const ctx = canvas.getContext('2d'); + if (!ctx) throw new Error('Canvas unavailable'); + ctx.imageSmoothingEnabled = true; + ctx.imageSmoothingQuality = 'high'; + ctx.fillStyle = '#101010'; + ctx.fillRect(0, 0, canvas.width, canvas.height); + // Source rectangle clipped to the frame; the destination shifts by the same amount. + const sx = Math.max(0, region.x); + const sy = Math.max(0, region.y); + const ex = Math.min(frame.width, region.x + region.width); + const ey = Math.min(frame.height, region.y + region.height); + if (ex > sx && ey > sy) { + const sw = ex - sx; + const sh = ey - sy; + ctx.drawImage( + frame.source, + sx, + sy, + sw, + sh, + (sx - region.x) * scale, + (sy - region.y) * scale, + sw * scale, + sh * scale, + ); + } + if (options.outline) { + const [x1, y1, x2, y2] = bounds; + ctx.strokeStyle = options.outline; + ctx.lineWidth = Math.max(1, Math.round(Math.max(canvas.width, canvas.height) / 160)); + ctx.strokeRect( + (Math.min(x1, x2) - region.x) * scale, + (Math.min(y1, y2) - region.y) * scale, + Math.abs(x2 - x1) * scale, + Math.abs(y2 - y1) * scale, + ); + } + const transform: ChipTransform = { + region, scale, width: canvas.width, height: canvas.height, + }; + // Upscaled crops hold few source pixels; keep them lossless so the little + // detail there is does not pick up compression noise. + const dataUrl = scale > 1 + ? canvas.toDataURL('image/png') + : canvas.toDataURL('image/jpeg', options.quality ?? 0.92); + return { dataUrl, transform }; +} diff --git a/client/dive-common/review/chipStore.ts b/client/dive-common/review/chipStore.ts new file mode 100644 index 000000000..4ebe84933 --- /dev/null +++ b/client/dive-common/review/chipStore.ts @@ -0,0 +1,211 @@ +/** + * Reactive cache of rendered chips for review items, filled by a small + * priority queue: the first box of every requested item ("primary") is + * rendered before any of the extra frames a track cycles through + * ("sequence"), so a page paints as fast as possible and then animates. + * + * Loads run with limited concurrency: a video dataset decodes frames by + * seeking one hidden element, and image sequences would otherwise fire a + * whole page of requests at once. + */ +import { ref, set, del } from 'vue'; +import type { FrameSource } from './frameSource'; +import { renderChip, ChipTransform, RenderedChip } from './chipRenderer'; +import type { ReviewItem } from './types'; + +export interface ChipStoreOptions { + padding: number; + size: number; + /** Chip width / height, matching the cells the chips are shown in. */ + aspect: number; + outline: string; +} + +export interface ChipStoreDeps { + /** Frame access for a dataset; null when the dataset cannot be cropped. */ + frameSourceFor(datasetId: string): FrameSource | null; + concurrency?: number; +} + +interface ChipJob { + item: ReviewItem; + /** Sequence slot to fill, or null for the primary chip. */ + slot: number | null; + generation: number; +} + +export type ChipSequence = Array; +export type ChipTransformSequence = Array; + +export function createChipStore(deps: ChipStoreDeps, initial: ChipStoreOptions) { + const concurrency = deps.concurrency ?? 4; + const chips = ref>({}); + const sequences = ref>({}); + const failures = ref>({}); + /** How each rendered chip maps to its frame, for overlays and editing. */ + const transforms = ref>({}); + const sequenceTransforms = ref>({}); + let options: ChipStoreOptions = { ...initial }; + let generation = 0; + let active = 0; + const primaryQueue: ChipJob[] = []; + const sequenceQueue: ChipJob[] = []; + /** Item keys with a primary job queued or running, and its generation. */ + const pendingPrimary = new Map(); + const sequenceQueued = new Map(); + + function reset() { + generation += 1; + chips.value = {}; + sequences.value = {}; + failures.value = {}; + transforms.value = {}; + sequenceTransforms.value = {}; + primaryQueue.length = 0; + sequenceQueue.length = 0; + pendingPrimary.clear(); + sequenceQueued.clear(); + } + + /** Re-render everything when the crop or resolution changes. */ + function setOptions(next: ChipStoreOptions) { + if (next.padding === options.padding && next.size === options.size + && next.aspect === options.aspect && next.outline === options.outline) { + return; + } + options = { ...next }; + reset(); + } + + async function render(job: ChipJob): Promise { + const source = deps.frameSourceFor(job.item.datasetId); + if (!source) throw new Error('Media for this dataset cannot be cropped'); + const frameRef = job.slot === null ? job.item.primary : job.item.frames[job.slot]; + const frame = await source.getFrame(frameRef.frame); + return renderChip(frame, frameRef.bounds, options); + } + + function complete(job: ChipJob, rendered: RenderedChip | null, error?: unknown) { + if (job.generation !== generation) return; + const { key } = job.item; + if (job.slot === null) { + if (rendered) { + set(chips.value, key, rendered.dataUrl); + set(transforms.value, key, rendered.transform); + } else { + set(failures.value, key, error instanceof Error ? error.message : 'Could not render this chip'); + } + } else if (rendered) { + const slots = sequences.value[key]; + if (slots) set(slots, job.slot, rendered.dataUrl); + const slotTransforms = sequenceTransforms.value[key]; + if (slotTransforms) set(slotTransforms, job.slot, rendered.transform); + } + } + + function finish(job: ChipJob) { + active -= 1; + if (job.slot === null && pendingPrimary.get(job.item.key) === job.generation) { + pendingPrimary.delete(job.item.key); + } + pump(); + } + + function start(job: ChipJob) { + active += 1; + render(job) + .then((rendered) => complete(job, rendered)) + .catch((err) => complete(job, null, err)) + .finally(() => finish(job)); + } + + function pump() { + while (active < concurrency) { + const job = primaryQueue.shift() ?? sequenceQueue.shift(); + if (!job) return; + start(job); + } + } + + /** Queue the first box of each item not already rendered or queued. */ + function ensurePrimary(items: readonly ReviewItem[]) { + items.forEach((item) => { + if (chips.value[item.key] || failures.value[item.key]) return; + if (pendingPrimary.get(item.key) === generation) return; + pendingPrimary.set(item.key, generation); + primaryQueue.push({ item, slot: null, generation }); + }); + pump(); + } + + /** + * Queue the cycling frames of track items. Call with just the visible + * page: every frame of a video dataset costs a seek. + */ + function ensureSequences(items: readonly ReviewItem[]) { + items.forEach((item) => { + if (item.frames.length < 2) return; + if (sequenceQueued.get(item.key) === generation) return; + sequenceQueued.set(item.key, generation); + // Fixed-size, null-filled so cells can show frames as they arrive. + set(sequences.value, item.key, item.frames.map((): string | null => null)); + set(sequenceTransforms.value, item.key, item.frames.map((): ChipTransform | null => null)); + item.frames.forEach((_, slot) => { + sequenceQueue.push({ item, slot, generation }); + }); + }); + pump(); + } + + /** Drop queued work that is no longer visible (rendered chips stay cached). */ + function trimQueues(visibleKeys: ReadonlySet) { + const keep = (job: ChipJob) => visibleKeys.has(job.item.key); + const droppedPrimary = primaryQueue.filter((job) => !keep(job)); + droppedPrimary.forEach((job) => { + if (pendingPrimary.get(job.item.key) === job.generation) pendingPrimary.delete(job.item.key); + }); + primaryQueue.splice(0, primaryQueue.length, ...primaryQueue.filter(keep)); + const droppedSequence = sequenceQueue.filter((job) => !keep(job)); + droppedSequence.forEach((job) => { + sequenceQueued.delete(job.item.key); + if (job.generation === generation) { + del(sequences.value, job.item.key); + del(sequenceTransforms.value, job.item.key); + } + }); + sequenceQueue.splice(0, sequenceQueue.length, ...sequenceQueue.filter(keep)); + } + + /** + * Forget an item's chips (its boxes changed) so the next ensure call + * renders them again. Queued work for the item is dropped too. + */ + function invalidate(key: string) { + del(chips.value, key); + del(failures.value, key); + del(transforms.value, key); + del(sequences.value, key); + del(sequenceTransforms.value, key); + pendingPrimary.delete(key); + sequenceQueued.delete(key); + primaryQueue.splice(0, primaryQueue.length, ...primaryQueue.filter((job) => job.item.key !== key)); + sequenceQueue.splice(0, sequenceQueue.length, ...sequenceQueue.filter((job) => job.item.key !== key)); + } + + return { + chips, + sequences, + failures, + transforms, + sequenceTransforms, + setOptions, + ensurePrimary, + ensureSequences, + trimQueues, + invalidate, + reset, + get options() { return options; }, + }; +} + +export type ChipStore = ReturnType; diff --git a/client/dive-common/review/frameSource.ts b/client/dive-common/review/frameSource.ts new file mode 100644 index 000000000..dd843b134 --- /dev/null +++ b/client/dive-common/review/frameSource.ts @@ -0,0 +1,234 @@ +/** + * Per-dataset access to decoded frames for chip cropping. Image sequences + * load their frame images directly; videos are decoded by a hidden + * HTMLVideoElement seeking to each requested frame. + */ +import type { DatasetConfig } from 'dive-common/apispec'; +import { frameToVideoTime } from 'vue-media-annotator/components/annotators/videoSeek'; + +/** Anything drawImage accepts, with its pixel size. */ +export interface DecodedFrame { + source: CanvasImageSource; + width: number; + height: number; +} + +export interface FrameSource { + /** Frames the dataset has; null when unknown (video before metadata loads). */ + frameCount: number | null; + getFrame(frame: number): Promise; + dispose(): void; +} + +export interface FrameSourceOptions { + /** Decoded frames kept per dataset. */ + cacheSize?: number; +} + +const DefaultCacheSize = 24; + +class FrameCache { + private entries = new Map(); + + private readonly limit: number; + + constructor(limit: number) { + this.limit = limit; + } + + get(frame: number): DecodedFrame | undefined { + const hit = this.entries.get(frame); + if (hit) { + // Re-insert so the map keeps least-recently-used order. + this.entries.delete(frame); + this.entries.set(frame, hit); + } + return hit; + } + + set(frame: number, decoded: DecodedFrame) { + this.entries.delete(frame); + this.entries.set(frame, decoded); + while (this.entries.size > this.limit) { + const oldest = this.entries.keys().next().value; + if (oldest === undefined) break; + this.entries.delete(oldest); + } + } + + clear() { + this.entries.clear(); + } +} + +export function loadImage(url: string): Promise { + return new Promise((resolve, reject) => { + const image = new Image(); + // Needed so the crop canvas is not tainted; both platforms serve media with CORS headers. + image.crossOrigin = 'anonymous'; + image.onload = () => resolve(image); + image.onerror = () => reject(new Error(`Could not load ${url}`)); + image.src = url; + }); +} + +/** Dedupes concurrent requests for the same frame in front of a loader. */ +function withCache( + cache: FrameCache, + load: (frame: number) => Promise, +): (frame: number) => Promise { + const inflight = new Map>(); + return (frame: number) => { + const cached = cache.get(frame); + if (cached) return Promise.resolve(cached); + const pending = inflight.get(frame); + if (pending) return pending; + const promise = load(frame).then((decoded) => { + cache.set(frame, decoded); + return decoded; + }).finally(() => inflight.delete(frame)); + inflight.set(frame, promise); + return promise; + }; +} + +function imageFrameSource(urlFor: (frame: number) => Promise, frameCount: number | null, cacheSize: number): FrameSource { + const cache = new FrameCache(cacheSize); + const getFrame = withCache(cache, async (frame) => { + const image = await loadImage(await urlFor(frame)); + return { source: image, width: image.naturalWidth, height: image.naturalHeight }; + }); + return { + frameCount, + getFrame, + dispose: () => cache.clear(), + }; +} + +const SeekTimeoutMs = 15000; + +/** + * One hidden video element per dataset; seeks are serialized because an + * element can only sit on one frame at a time. Each decoded frame is copied + * to its own canvas so the cache survives the next seek. + */ +function videoFrameSource(config: DatasetConfig, cacheSize: number): FrameSource { + const cache = new FrameCache(cacheSize); + let video: HTMLVideoElement | null = null; + let metadata: Promise | null = null; + let queue: Promise = Promise.resolve(); + let disposed = false; + + function element(): Promise { + if (metadata) return metadata; + metadata = new Promise((resolve, reject) => { + const el = document.createElement('video'); + el.crossOrigin = 'anonymous'; + el.muted = true; + el.preload = 'auto'; + el.playsInline = true; + el.onloadedmetadata = () => resolve(el); + el.onerror = () => reject(new Error(`Could not open video for ${config.name}`)); + el.src = config.videoUrl || ''; + video = el; + }); + return metadata; + } + + function seekTo(el: HTMLVideoElement, time: number): Promise { + return new Promise((resolve, reject) => { + let settled = false; + const timer = window.setTimeout(() => { + if (settled) return; + settled = true; + cleanup(); + reject(new Error('Timed out seeking the video')); + }, SeekTimeoutMs); + function cleanup() { + el.removeEventListener('seeked', onSeeked); + el.removeEventListener('error', onError); + window.clearTimeout(timer); + } + function onSeeked() { + if (settled) return; + settled = true; + cleanup(); + resolve(); + } + function onError() { + if (settled) return; + settled = true; + cleanup(); + reject(new Error('The video failed while seeking')); + } + el.addEventListener('seeked', onSeeked); + el.addEventListener('error', onError); + // Seeking to the time the element already sits at fires no event. + if (Math.abs(el.currentTime - time) < 1e-6 && el.readyState >= 2) { + onSeeked(); + return; + } + // eslint-disable-next-line no-param-reassign + el.currentTime = time; + }); + } + + const getFrame = withCache(cache, (frame) => { + const run = queue.then(async () => { + if (disposed) throw new Error('Frame source disposed'); + const el = await element(); + await seekTo(el, frameToVideoTime(frame, config.fps, config.originalFps ?? null)); + const canvas = document.createElement('canvas'); + canvas.width = el.videoWidth; + canvas.height = el.videoHeight; + const ctx = canvas.getContext('2d'); + if (!ctx) throw new Error('Canvas unavailable'); + ctx.drawImage(el, 0, 0); + return { source: canvas, width: canvas.width, height: canvas.height } as DecodedFrame; + }); + // Failures must not wedge the queue for later frames. + queue = run.catch(() => undefined); + return run; + }); + + return { + frameCount: null, + getFrame, + dispose: () => { + disposed = true; + cache.clear(); + if (video) { + video.removeAttribute('src'); + video.load(); + video = null; + } + metadata = null; + }, + }; +} + +/** + * Pick the loader for a dataset, or throw for media the review grid cannot + * crop (tiled large images, multicamera parents). + */ +export function createFrameSource(config: DatasetConfig, options: FrameSourceOptions = {}): FrameSource { + const cacheSize = options.cacheSize ?? DefaultCacheSize; + if (config.type === 'large-image') { + throw new Error('Tiled large-image datasets cannot be reviewed as chips yet'); + } + if (config.type === 'multi') { + throw new Error('Review the cameras of a multicamera dataset individually'); + } + if (config.type === 'video') { + if (config.videoUrl) { + return videoFrameSource(config, cacheSize); + } + throw new Error('This video has no playable media'); + } + const { imageData } = config; + return imageFrameSource(async (frame) => { + const entry = imageData[frame]; + if (!entry) throw new Error(`No image for frame ${frame}`); + return entry.url; + }, imageData.length, cacheSize); +} diff --git a/client/dive-common/review/gridSettings.spec.ts b/client/dive-common/review/gridSettings.spec.ts new file mode 100644 index 000000000..be1a0abb3 --- /dev/null +++ b/client/dive-common/review/gridSettings.spec.ts @@ -0,0 +1,33 @@ +import { cellScaleFor, clampGrid } from './gridSettings'; +import { DEFAULT_REVIEW_GRID } from './types'; + +describe('clampGrid', () => { + it('rounds and clamps columns and rows and clamps padding', () => { + expect(clampGrid({ + ...DEFAULT_REVIEW_GRID, columns: 40, rows: 0.4, padding: -2, + })).toMatchObject({ + columns: 12, rows: 1, padding: 0, + }); + expect(clampGrid({ + ...DEFAULT_REVIEW_GRID, columns: 3.6, rows: 2.2, padding: 0.5, + })).toMatchObject({ + columns: 4, rows: 2, padding: 0.5, + }); + }); + + it('falls back to defaults for non-numbers', () => { + expect(clampGrid({ ...DEFAULT_REVIEW_GRID, columns: Number.NaN, rows: Infinity })).toMatchObject({ + columns: DEFAULT_REVIEW_GRID.columns, rows: DEFAULT_REVIEW_GRID.rows, + }); + }); +}); + +describe('cellScaleFor', () => { + it('grows footer text as the grid shows fewer cells, within limits', () => { + expect(cellScaleFor(5, 4)).toBeCloseTo(1.12, 2); + expect(cellScaleFor(3, 3)).toBe(1.35); + expect(cellScaleFor(1, 1)).toBe(1.35); + expect(cellScaleFor(8, 6)).toBe(1); + expect(cellScaleFor(12, 10)).toBe(1); + }); +}); diff --git a/client/dive-common/review/gridSettings.ts b/client/dive-common/review/gridSettings.ts new file mode 100644 index 000000000..cc10bdbb7 --- /dev/null +++ b/client/dive-common/review/gridSettings.ts @@ -0,0 +1,60 @@ +/** + * Grid presentation settings (columns, rows, context margin) shared by every + * chip grid in the app and remembered per browser. + */ +import { reactive, watch } from 'vue'; +import { DEFAULT_REVIEW_GRID, REVIEW_GRID_LIMITS, ReviewGridSettings } from './types'; + +/** + * Size factor for a cell's footer text: a 5x4 grid reads about a tenth + * larger than the base size, a 3x3 grid about a third larger, and dense + * grids stay at the base size. + */ +export function cellScaleFor(columns: number, rows: number): number { + const cells = Math.max(1, columns * rows); + return Math.round(Math.min(1.35, Math.max(1, 5 / Math.sqrt(cells))) * 100) / 100; +} + +const GRID_STORAGE_KEY = 'dive.review.grid'; + +export function loadGridSettings(): ReviewGridSettings { + try { + const raw = window.localStorage.getItem(GRID_STORAGE_KEY); + if (raw) { + const parsed = JSON.parse(raw); + if (parsed && typeof parsed === 'object') { + return { ...DEFAULT_REVIEW_GRID, ...parsed }; + } + } + } catch { + // Storage may be unavailable; defaults are fine. + } + return { ...DEFAULT_REVIEW_GRID }; +} + +export function storeGridSettings(grid: ReviewGridSettings) { + try { + window.localStorage.setItem(GRID_STORAGE_KEY, JSON.stringify(grid)); + } catch { + // Ignore storage failures. + } +} + +export function clampGrid(grid: ReviewGridSettings): ReviewGridSettings { + const clamp = (value: number, [min, max]: readonly [number, number], fallback: number) => ( + Number.isFinite(value) ? Math.min(max, Math.max(min, value)) : fallback + ); + return { + ...grid, + columns: Math.round(clamp(grid.columns, REVIEW_GRID_LIMITS.columns, DEFAULT_REVIEW_GRID.columns)), + rows: Math.round(clamp(grid.rows, REVIEW_GRID_LIMITS.rows, DEFAULT_REVIEW_GRID.rows)), + padding: clamp(grid.padding, REVIEW_GRID_LIMITS.padding, DEFAULT_REVIEW_GRID.padding), + }; +} + +/** A reactive settings object seeded from storage and written back on change. */ +export function usePersistentGridSettings(): ReviewGridSettings { + const grid = reactive(clampGrid(loadGridSettings())); + watch(grid, () => storeGridSettings({ ...grid }), { deep: true }); + return grid; +} diff --git a/client/dive-common/review/reviewItems.spec.ts b/client/dive-common/review/reviewItems.spec.ts new file mode 100644 index 000000000..fb80296e8 --- /dev/null +++ b/client/dive-common/review/reviewItems.spec.ts @@ -0,0 +1,256 @@ +import type { TrackData } from 'vue-media-annotator/track'; +import { + attributeMatches, + buildReviewItems, + collectAttributeKeys, + collectTypes, + cycleIntervalFor, + frameGeometry, + groupReviewItems, + interpolateBounds, + matchTypePair, + sampleFrames, + sortReviewItems, +} from './reviewItems'; +import { DEFAULT_REVIEW_QUERY } from './types'; + +function track( + id: number, + pairs: [string, number][], + frames: number[], + extra: Partial = {}, +): TrackData { + return { + id, + begin: Math.min(...frames), + end: Math.max(...frames), + confidencePairs: pairs, + attributes: {}, + features: frames.map((frame) => ({ + frame, keyframe: true, bounds: [frame, 0, frame + 10, 10], + })), + ...extra, + }; +} + +describe('sampleFrames', () => { + it('keeps a single detection as one frame', () => { + const [only] = track(1, [['a', 1]], [3]).features; + expect(sampleFrames([only], 8)).toEqual([{ frame: 3, bounds: [3, 0, 13, 10] }]); + }); + + it('samples evenly including both ends and never repeats a frame', () => { + const { features } = track(1, [['a', 1]], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + const frames = sampleFrames(features, 4).map((f) => f.frame); + expect(frames).toEqual([0, 3, 7, 10]); + expect(sampleFrames(features, 50)).toHaveLength(11); + }); +}); + +describe('matchTypePair', () => { + it('uses the top pair when no type is requested', () => { + expect(matchTypePair([['fish', 0.4], ['shark', 0.9]], '', 0.5)).toEqual(['shark', 0.9]); + expect(matchTypePair([['fish', 0.4]], '', 0.5)).toBeNull(); + }); + + it('matches the named type against the threshold', () => { + expect(matchTypePair([['fish', 0.4], ['shark', 0.9]], 'fish', 0.3)).toEqual(['fish', 0.4]); + expect(matchTypePair([['fish', 0.4], ['shark', 0.9]], 'fish', 0.5)).toBeNull(); + expect(matchTypePair([['shark', 0.9]], 'fish', 0)).toBeNull(); + }); + + it('only shows untyped tracks when everything is requested', () => { + expect(matchTypePair([], '', 0)).toEqual(['', 0]); + expect(matchTypePair([], '', 0.1)).toBeNull(); + }); +}); + +describe('attributeMatches', () => { + it('requires presence, then compares as text ignoring case', () => { + expect(attributeMatches(undefined, '')).toBe(false); + expect(attributeMatches('Yes', '')).toBe(true); + expect(attributeMatches('Yes', 'yes')).toBe(true); + expect(attributeMatches(3, '3')).toBe(true); + expect(attributeMatches(true, 'false')).toBe(false); + }); +}); + +describe('buildReviewItems', () => { + const tracks = [ + track(2, [['fish', 0.8], ['shark', 0.2]], [10, 11, 12]), + track(1, [['shark', 0.95]], [5]), + track(3, [['fish', 0.05]], [0]), + { ...track(4, [['fish', 0.9]], [7]), features: [{ frame: 7, keyframe: false }] }, + ]; + + it('filters by type and threshold and orders by track id', () => { + const items = buildReviewItems('ds', tracks, { ...DEFAULT_REVIEW_QUERY, type: 'fish', threshold: 0.1 }, 8); + expect(items.map((i) => i.trackId)).toEqual([2]); + expect(items[0]).toMatchObject({ + key: 'ds#2', type: 'fish', confidence: 0.8, keyframeCount: 3, + }); + expect(items[0].primary.frame).toBe(10); + expect(items[0].frames).toHaveLength(3); + }); + + it('shows every class above the threshold when no type is picked', () => { + const items = buildReviewItems('ds', tracks, { ...DEFAULT_REVIEW_QUERY, type: '', threshold: 0.5 }, 8); + expect(items.map((i) => [i.trackId, i.type])).toEqual([[1, 'shark'], [2, 'fish']]); + }); + + it('finds track and detection attributes', () => { + const withAttributes = [ + track(1, [['fish', 1]], [0, 1], { attributes: { verified: true } }), + { + ...track(2, [['fish', 1]], [4, 5, 6]), + features: [ + { frame: 4, keyframe: true, bounds: [0, 0, 1, 1] as [number, number, number, number] }, + { + frame: 5, keyframe: true, bounds: [0, 0, 1, 1] as [number, number, number, number], attributes: { occluded: 'partial' }, + }, + { + frame: 6, keyframe: true, bounds: [0, 0, 1, 1] as [number, number, number, number], attributes: { occluded: 'partial' }, + }, + ], + }, + ]; + const query = { ...DEFAULT_REVIEW_QUERY, mode: 'attribute' as const }; + const verified = buildReviewItems('ds', withAttributes, { ...query, attributeKey: 'verified' }, 8); + expect(verified.map((i) => i.trackId)).toEqual([1]); + expect(verified[0].matchedAttribute).toEqual({ key: 'verified', value: true, scope: 'track' }); + + const occluded = buildReviewItems('ds', withAttributes, { ...query, attributeKey: 'occluded', attributeValue: 'PARTIAL' }, 8); + expect(occluded).toHaveLength(1); + expect(occluded[0]).toMatchObject({ key: 'ds#2@5', trackId: 2, type: 'fish' }); + expect(occluded[0].primary.frame).toBe(5); + expect(occluded[0].frames.map((f) => f.frame)).toEqual([5, 6]); + + const trackOnly = buildReviewItems('ds', withAttributes, { ...query, attributeKey: 'occluded', attributeScope: 'track' }, 8); + expect(trackOnly).toHaveLength(0); + }); +}); + +describe('sortReviewItems', () => { + const items = [ + ...buildReviewItems('b', [track(1, [['a', 0.5]], [9]), track(2, [['a', 0.9]], [3])], DEFAULT_REVIEW_QUERY, 8), + ...buildReviewItems('a', [track(7, [['a', 0.7]], [1])], DEFAULT_REVIEW_QUERY, 8), + ]; + + it('orders by dataset selection order, confidence or frame', () => { + const keys = (order: Parameters[1]) => ( + sortReviewItems(items, order, ['a', 'b']).map((i) => i.key) + ); + expect(keys('dataset')).toEqual(['a#7', 'b#1', 'b#2']); + expect(keys('confidence-asc')).toEqual(['b#1', 'a#7', 'b#2']); + expect(keys('confidence-desc')).toEqual(['b#2', 'a#7', 'b#1']); + expect(keys('frame')).toEqual(['a#7', 'b#2', 'b#1']); + }); +}); + +describe('vocabularies', () => { + it('collects sorted types and attribute keys', () => { + const tracks = [ + track(1, [['zeta', 0.1], ['alpha', 0.9]], [0], { attributes: { trackAttr: 1 } }), + { + ...track(2, [['beta', 1]], [0]), + features: [{ + frame: 0, keyframe: true, bounds: [0, 0, 1, 1] as [number, number, number, number], attributes: { detAttr: 'x' }, + }], + }, + ]; + expect(collectTypes(tracks)).toEqual(['alpha', 'beta', 'zeta']); + expect(collectAttributeKeys(tracks, { + defined: { + belongs: 'track', datatype: 'text', name: 'defined', key: 'track_defined', + }, + })).toEqual(['defined', 'detAttr', 'trackAttr']); + }); +}); + +describe('frameGeometry', () => { + it('reads polygons and head/tail points from the GeoJSON features', () => { + const geometry = frameGeometry({ + frame: 2, + bounds: [0, 0, 10, 10], + geometry: { + type: 'FeatureCollection', + features: [ + { + type: 'Feature', + properties: { key: '' }, + geometry: { type: 'Polygon', coordinates: [[[1, 1], [9, 1], [9, 9], [1, 1]]] }, + }, + { type: 'Feature', properties: { key: 'head' }, geometry: { type: 'Point', coordinates: [3, 4] } }, + { type: 'Feature', properties: { key: 'tail' }, geometry: { type: 'Point', coordinates: [7, 8] } }, + { type: 'Feature', properties: { key: 'HeadTails' }, geometry: { type: 'LineString', coordinates: [[3, 4], [7, 8]] } }, + ], + }, + }); + expect(geometry).toEqual({ + polygons: [[[1, 1], [9, 1], [9, 9]]], + head: [3, 4], + tail: [7, 8], + }); + }); + + it('falls back to the feature head/tail fields and leaves plain boxes bare', () => { + expect(frameGeometry({ frame: 0, bounds: [0, 0, 1, 1], head: [1, 2] })).toEqual({ head: [1, 2] }); + expect(frameGeometry({ frame: 0, bounds: [0, 0, 1, 1] })).toEqual({}); + }); +}); + +describe('cycleIntervalFor', () => { + it('plays consecutive frames at the dataset rate and sparse samples proportionally slower', () => { + const consecutive = [0, 1, 2, 3].map((frame) => ({ frame, bounds: [0, 0, 1, 1] as [number, number, number, number] })); + expect(cycleIntervalFor(consecutive, 10, 400)).toBe(100); + const sparse = [0, 30, 60, 90].map((frame) => ({ frame, bounds: [0, 0, 1, 1] as [number, number, number, number] })); + expect(cycleIntervalFor(sparse, 30, 400)).toBe(1000); + expect(cycleIntervalFor(sparse, 1, 400)).toBe(2000); + expect(cycleIntervalFor(consecutive, 120, 400)).toBe(33); + }); + + it('falls back when the rate is unknown or there is nothing to cycle', () => { + const one = [{ frame: 5, bounds: [0, 0, 1, 1] as [number, number, number, number] }]; + expect(cycleIntervalFor(one, 30, 400)).toBe(400); + expect(cycleIntervalFor([...one, { frame: 6, bounds: [0, 0, 1, 1] }], 0, 400)).toBe(400); + }); +}); + +describe('interpolateBounds', () => { + it('holds the nearest box at the ends and interpolates between keyframes', () => { + const t = track(1, [['fish', 1]], [0, 10]); + expect(interpolateBounds(t, 5)).toEqual([5, 0, 15, 10]); + expect(interpolateBounds(t, -3)).toEqual([0, 0, 10, 10]); + expect(interpolateBounds(t, 20)).toEqual([10, 0, 20, 10]); + expect(interpolateBounds(t, 10)).toEqual([10, 0, 20, 10]); + expect(interpolateBounds({ ...t, features: [] }, 3)).toBeNull(); + }); +}); + +describe('groupReviewItems', () => { + it('joins a track across the cameras of a rig with aligned frames and box-less gaps', () => { + const left = track(7, [['fish', 1]], [0, 4]); + const right = track(7, [['fish', 1]], [4, 8]); + const query = { ...DEFAULT_REVIEW_QUERY, threshold: 0 }; + const items = [ + ...buildReviewItems('rig/right', [right], query, 8), + ...buildReviewItems('rig/left', [left], query, 8), + ...buildReviewItems('solo', [track(1, [['fish', 1]], [2])], query, 8), + ]; + const membership = (id: string) => (id.startsWith('rig/') + ? { parent: 'rig', camera: id.slice(4), rank: id === 'rig/left' ? 0 : 1 } + : undefined); + const tracks: Record = { 'rig/left': left, 'rig/right': right }; + const entries = groupReviewItems(items, membership, (item) => tracks[item.datasetId], 8); + + expect(entries.map((e) => e.key)).toEqual(['rig#7', 'solo#1']); + const rig = entries[0]; + expect(rig.labels).toEqual(['left', 'right']); + expect(rig.items.map((i) => i.datasetId)).toEqual(['rig/left', 'rig/right']); + // Both cameras show frames 0, 4 and 8; the sides without a detection are interpolated. + expect(rig.items[0].frames.map((f) => [f.frame, f.missing ?? false])).toEqual([[0, false], [4, false], [8, true]]); + expect(rig.items[1].frames.map((f) => [f.frame, f.missing ?? false])).toEqual([[0, true], [4, false], [8, false]]); + expect(rig.items[0].frames[2].bounds).toEqual([4, 0, 14, 10]); + expect(entries[1].labels).toEqual(['']); + }); +}); diff --git a/client/dive-common/review/reviewItems.ts b/client/dive-common/review/reviewItems.ts new file mode 100644 index 000000000..6574bbcec --- /dev/null +++ b/client/dive-common/review/reviewItems.ts @@ -0,0 +1,387 @@ +/** + * Pure helpers that turn a dataset's tracks into review grid items for a + * query, plus the type/attribute vocabularies the query controls offer. + */ +import type { TrackData, Feature } from 'vue-media-annotator/track'; +import type { StringKeyObject } from 'vue-media-annotator/BaseAnnotation'; +import type { Attribute } from 'vue-media-annotator/use/AttributeTypes'; +import { compareTypeNames } from 'dive-common/typeHierarchy'; +import type { RectBounds } from 'vue-media-annotator/utils'; +import type { + ReviewEntry, ReviewFrameGeometry, ReviewFrameRef, ReviewItem, ReviewPolygon, ReviewQuery, ReviewSortOrder, +} from './types'; + +function isPoint(value: unknown): value is [number, number] { + return Array.isArray(value) && value.length >= 2 + && Number.isFinite(value[0]) && Number.isFinite(value[1]); +} + +/** + * Polygons and head/tail points of a keyframe. Points come from the + * GeoJSON features keyed "head"/"tail" (how DIVE stores them), falling back + * to the feature's own head/tail fields. + */ +export function frameGeometry(feature: Feature): ReviewFrameGeometry { + const geometry: ReviewFrameGeometry = {}; + const polygons: ReviewPolygon[] = []; + (feature.geometry?.features || []).forEach((geo) => { + const key = (geo.properties as { key?: unknown } | null)?.key; + if (geo.geometry.type === 'Polygon') { + const ring = (geo.geometry.coordinates[0] || []).filter(isPoint).map(([x, y]) => [x, y] as [number, number]); + if (ring.length > 1 && ring[0][0] === ring[ring.length - 1][0] && ring[0][1] === ring[ring.length - 1][1]) { + ring.pop(); + } + if (ring.length >= 3) polygons.push(ring); + } else if (geo.geometry.type === 'Point' && (key === 'head' || key === 'tail')) { + const point = geo.geometry.coordinates; + if (isPoint(point)) geometry[key] = [point[0], point[1]]; + } + }); + if (!geometry.head && isPoint(feature.head)) geometry.head = [feature.head[0], feature.head[1]]; + if (!geometry.tail && isPoint(feature.tail)) geometry.tail = [feature.tail[0], feature.tail[1]]; + if (polygons.length) geometry.polygons = polygons; + return geometry; +} + +/** A frame reference for a keyframe, carrying its extra geometry when present. */ +export function frameRefFor(feature: Feature): ReviewFrameRef | null { + if (!feature.bounds) return null; + return { frame: feature.frame, bounds: feature.bounds, ...frameGeometry(feature) }; +} + +/** Keyframes carrying a box, in frame order. */ +export function boxedFeatures(track: TrackData): Feature[] { + return track.features + .filter((f) => !!f.bounds) + .sort((a, b) => a.frame - b.frame); +} + +/** Up to `max` boxes evenly sampled along `features` (which must be in frame order). */ +export function sampleFrames(features: Feature[], max: number): ReviewFrameRef[] { + if (features.length === 0) return []; + const count = Math.max(1, Math.min(max, features.length)); + const chosen: ReviewFrameRef[] = []; + const seen = new Set(); + for (let i = 0; i < count; i += 1) { + const index = count === 1 ? 0 : Math.round((i * (features.length - 1)) / (count - 1)); + const feature = features[index]; + const ref = frameRefFor(feature); + if (ref && !seen.has(feature.frame)) { + seen.add(feature.frame); + chosen.push(ref); + } + } + return chosen; +} + +/** + * Milliseconds between the sampled frames of a track so that cycling + * through them plays at the dataset's real-time rate: consecutive frames + * advance every 1/fps seconds, and sparser samples wait proportionally + * longer. Clamped so a loop is neither a strobe nor a slideshow. + */ +export function cycleIntervalFor(frames: readonly ReviewFrameRef[], fps: number, fallbackMs: number): number { + if (frames.length < 2 || !(fps > 0)) return fallbackMs; + const span = frames[frames.length - 1].frame - frames[0].frame; + const stride = Math.max(1, span / (frames.length - 1)); + return Math.min(2000, Math.max(33, Math.round((stride / fps) * 1000))); +} + +/** The pair a type query matches on, or null when the track does not qualify. */ +export function matchTypePair( + pairs: readonly (readonly [string, number])[], + type: string, + threshold: number, +): [string, number] | null { + if (pairs.length === 0) { + // Untyped annotations only show up when every type is requested. + return type === '' && threshold <= 0 ? ['', 0] : null; + } + // Stored pairs are normally sorted by confidence, but files are not guaranteed to be. + const candidates = type === '' ? pairs : pairs.filter(([name]) => name === type); + const best = candidates.reduce( + (acc, pair) => (acc === null || pair[1] > acc[1] ? pair : acc), + null, + ); + if (!best || best[1] < threshold) return null; + return [best[0], best[1]]; +} + +function ownAttributes(attributes: StringKeyObject | undefined): [string, unknown][] { + if (!attributes) return []; + return Object.entries(attributes).filter(([key]) => key !== 'userAttributes'); +} + +/** Attribute values compare as strings, case-insensitively; empty wanted means "present". */ +export function attributeMatches(value: unknown, wanted: string): boolean { + if (value === undefined || value === null) return false; + if (wanted === '') return true; + const asText = Array.isArray(value) ? value.map(String).join(',') : String(value); + return asText.toLowerCase() === wanted.trim().toLowerCase(); +} + +function trackTopPair(track: TrackData): [string, number] { + const top = track.confidencePairs.reduce( + (acc, pair) => (acc === null || pair[1] > acc[1] ? pair : acc), + null, + ); + return top ? [top[0], top[1]] : ['', 0]; +} + +function attributeItem( + datasetId: string, + track: TrackData, + query: ReviewQuery, + maxSequenceFrames: number, +): ReviewItem | null { + const key = query.attributeKey.trim(); + if (!key) return null; + const features = boxedFeatures(track); + if (features.length === 0) return null; + const [type, confidence] = trackTopPair(track); + const base = { + datasetId, + trackId: track.id, + keyframeCount: features.length, + type, + confidence, + }; + if (query.attributeScope !== 'detection') { + const hit = ownAttributes(track.attributes).find( + ([name, value]) => name === key && attributeMatches(value, query.attributeValue), + ); + if (hit) { + const frames = sampleFrames(features, maxSequenceFrames); + return { + ...base, + key: `${datasetId}#${track.id}`, + primary: frames[0], + frames, + matchedAttribute: { key, value: hit[1], scope: 'track' }, + }; + } + } + if (query.attributeScope !== 'track') { + const matching = features.filter((f) => ownAttributes(f.attributes).some( + ([name, value]) => name === key && attributeMatches(value, query.attributeValue), + )); + if (matching.length > 0) { + const frames = sampleFrames(matching, maxSequenceFrames); + const hit = ownAttributes(matching[0].attributes).find(([name]) => name === key); + return { + ...base, + key: `${datasetId}#${track.id}@${matching[0].frame}`, + primary: frames[0], + frames, + matchedAttribute: { key, value: hit ? hit[1] : undefined, scope: 'detection' }, + }; + } + } + return null; +} + +function typeItem( + datasetId: string, + track: TrackData, + query: ReviewQuery, + maxSequenceFrames: number, +): ReviewItem | null { + const pair = matchTypePair(track.confidencePairs, query.type, query.threshold); + if (!pair) return null; + const features = boxedFeatures(track); + if (features.length === 0) return null; + const frames = sampleFrames(features, maxSequenceFrames); + return { + key: `${datasetId}#${track.id}`, + datasetId, + trackId: track.id, + primary: frames[0], + frames, + keyframeCount: features.length, + type: pair[0], + confidence: pair[1], + }; +} + +/** Every grid item one dataset contributes to a query, in track id order. */ +export function buildReviewItems( + datasetId: string, + tracks: Iterable, + query: ReviewQuery, + maxSequenceFrames: number, +): ReviewItem[] { + const items: ReviewItem[] = []; + Array.from(tracks) + .sort((a, b) => a.id - b.id) + .forEach((track) => { + const item = query.mode === 'attribute' + ? attributeItem(datasetId, track, query, maxSequenceFrames) + : typeItem(datasetId, track, query, maxSequenceFrames); + if (item) items.push(item); + }); + return items; +} + +export function sortReviewItems( + items: ReviewItem[], + order: ReviewSortOrder, + datasetOrder: readonly string[], +): ReviewItem[] { + const rank = new Map(datasetOrder.map((id, index) => [id, index])); + const byDataset = (a: ReviewItem, b: ReviewItem) => ( + (rank.get(a.datasetId) ?? Infinity) - (rank.get(b.datasetId) ?? Infinity) + ); + const sorted = [...items]; + switch (order) { + case 'confidence-asc': + sorted.sort((a, b) => a.confidence - b.confidence || byDataset(a, b) || a.trackId - b.trackId); + break; + case 'confidence-desc': + sorted.sort((a, b) => b.confidence - a.confidence || byDataset(a, b) || a.trackId - b.trackId); + break; + case 'frame': + sorted.sort((a, b) => byDataset(a, b) || a.primary.frame - b.primary.frame || a.trackId - b.trackId); + break; + default: + sorted.sort((a, b) => byDataset(a, b) || a.trackId - b.trackId); + } + return sorted; +} + +/** Which multicamera parent and camera a dataset id belongs to, if any. */ +export interface CameraMembership { + parent: string; + camera: string; + /** Position of the camera in the rig's display order. */ + rank: number; +} + +/** + * Box a track would have on `frame` in one camera, interpolated between + * its nearest keyframes with boxes (held at the ends), or null when the + * track has no boxes there at all. + */ +export function interpolateBounds(track: TrackData, frame: number): RectBounds | null { + const features = boxedFeatures(track); + if (features.length === 0) return null; + const exact = features.find((f) => f.frame === frame); + if (exact?.bounds) return exact.bounds; + const before = [...features].reverse().find((f) => f.frame < frame); + const after = features.find((f) => f.frame > frame); + if (before?.bounds && after?.bounds) { + const t = (frame - before.frame) / (after.frame - before.frame); + return before.bounds.map((v, i) => v + ((after.bounds as RectBounds)[i] - v) * t) as RectBounds; + } + return (before?.bounds ?? after?.bounds) ?? null; +} + +/** + * Give the items of a multicamera entry the same frames: the union of their + * keyframes sampled once, with a camera that lacks a detection on a frame + * getting an interpolated, box-less reference there. + */ +export function alignCameraFrames( + items: ReviewItem[], + trackOf: (item: ReviewItem) => TrackData | undefined, + maxSequenceFrames: number, +): ReviewItem[] { + if (items.length < 2) return items; + const byFrame = new Map(); + items.forEach((item) => { + const track = trackOf(item); + if (!track) return; + boxedFeatures(track).forEach((feature) => { + if (!byFrame.has(feature.frame)) byFrame.set(feature.frame, feature); + }); + }); + const union = Array.from(byFrame.values()).sort((a, b) => a.frame - b.frame); + // Detection matches keep their own frame first so the query hit stays visible. + const sampled = sampleFrames(union, maxSequenceFrames).map((ref) => ref.frame); + const anchor = items.find((item) => item.key.includes('@'))?.primary.frame; + if (anchor !== undefined && !sampled.includes(anchor)) sampled.unshift(anchor); + + return items.map((item) => { + const track = trackOf(item); + if (!track) return item; + const frames: ReviewFrameRef[] = []; + sampled.forEach((frame) => { + const feature = track.features.find((f) => f.frame === frame && f.bounds); + if (feature) { + const ref = frameRefFor(feature); + if (ref) frames.push(ref); + return; + } + const bounds = interpolateBounds(track, frame); + if (bounds) frames.push({ frame, bounds, missing: true }); + }); + if (frames.length === 0) return item; + return { + ...item, primary: frames[0], frames, keyframeCount: boxedFeatures(track).length, + }; + }); +} + +/** + * Group items into grid entries: one per track, holding an item per camera + * the track appears in (in rig order) for multicamera datasets. + */ +export function groupReviewItems( + items: readonly ReviewItem[], + membershipOf: (datasetId: string) => CameraMembership | undefined, + trackOf: (item: ReviewItem) => TrackData | undefined, + maxSequenceFrames: number, +): ReviewEntry[] { + const entries: ReviewEntry[] = []; + const byKey = new Map(); + items.forEach((item) => { + const membership = membershipOf(item.datasetId); + if (!membership) { + entries.push({ key: item.key, items: [item], labels: [''] }); + return; + } + const key = item.key.replace(item.datasetId, membership.parent); + let group = byKey.get(key); + if (!group) { + group = { items: [], ranks: [], labels: [] }; + byKey.set(key, group); + entries.push({ key, items: group.items, labels: group.labels }); + } + // Keep cameras in rig order whatever order the items arrived in. + let at = group.ranks.findIndex((rank) => rank > membership.rank); + if (at < 0) at = group.ranks.length; + group.items.splice(at, 0, item); + group.ranks.splice(at, 0, membership.rank); + group.labels.splice(at, 0, membership.camera); + }); + return entries.map((entry) => (entry.items.length > 1 + ? { ...entry, items: alignCameraFrames(entry.items, trackOf, maxSequenceFrames) } + : entry)); +} + +/** Every type named by any confidence pair, in type order. */ +export function collectTypes(tracks: Iterable): string[] { + const types = new Set(); + Array.from(tracks).forEach((track) => { + track.confidencePairs.forEach(([type]) => { if (type) types.add(type); }); + }); + return Array.from(types).sort(compareTypeNames); +} + +/** + * Attribute keys the query control can offer: those defined in the dataset + * configuration plus any actually present on tracks or detections. + */ +export function collectAttributeKeys( + tracks: Iterable, + definitions: Readonly> | undefined, +): string[] { + const keys = new Set(); + Object.values(definitions || {}).forEach((attribute) => keys.add(attribute.name)); + Array.from(tracks).forEach((track) => { + ownAttributes(track.attributes).forEach(([key]) => keys.add(key)); + track.features.forEach((feature) => { + ownAttributes(feature.attributes).forEach(([key]) => keys.add(key)); + }); + }); + return Array.from(keys).sort((a, b) => a.localeCompare(b)); +} diff --git a/client/dive-common/review/types.ts b/client/dive-common/review/types.ts new file mode 100644 index 000000000..1f5a4442e --- /dev/null +++ b/client/dive-common/review/types.ts @@ -0,0 +1,130 @@ +/** + * Review mode: contract shared by the review page, the chip loader and the + * platform shells. The page shows many detections at once as cropped chips + * so a user can audit and correct their types (or find them by attribute) + * without opening every sequence in the viewer. + */ +import type { AnnotationId } from 'vue-media-annotator/BaseAnnotation'; +import type { RectBounds } from 'vue-media-annotator/utils'; + +export type ReviewQueryMode = 'type' | 'attribute'; + +/** Where an attribute query looks for the attribute. */ +export type ReviewAttributeScope = 'any' | 'track' | 'detection'; + +export interface ReviewQuery { + mode: ReviewQueryMode; + /** Type mode: the class to show; empty for every class. */ + type: string; + /** Type mode: minimum confidence of the matching pair (0 shows everything). */ + threshold: number; + /** Attribute mode: the attribute key to look for. */ + attributeKey: string; + /** Attribute mode: required value (string compared); empty just requires presence. */ + attributeValue: string; + attributeScope: ReviewAttributeScope; +} + +export type ReviewSortOrder = 'dataset' | 'confidence-asc' | 'confidence-desc' | 'frame'; + +/** A polygon outline in image coordinates: [[x, y], ...] without a closing repeat. */ +export type ReviewPolygon = [number, number][]; + +/** Extra geometry a detection may carry besides its box. */ +export interface ReviewFrameGeometry { + polygons?: ReviewPolygon[]; + head?: [number, number]; + tail?: [number, number]; +} + +/** One box of a track shown in a chip, in the dataset's own frame numbers. */ +export interface ReviewFrameRef extends ReviewFrameGeometry { + frame: number; + bounds: RectBounds; + /** + * The track has no detection on this frame in this camera; `bounds` is + * interpolated from its neighbours so the chip can still be cropped there, + * and no box is drawn. + */ + missing?: boolean; +} + +/** + * One grid entry: a track (or one detection of it) in one dataset. Built + * once per query run; the live type is read from the review service so + * edits show without the grid reshuffling. + */ +export interface ReviewItem { + /** Stable within a query run: `${datasetId}#${trackId}` or with `@frame`. */ + key: string; + datasetId: string; + trackId: AnnotationId; + /** The box shown first: the matched detection, or the track's first keyframe. */ + primary: ReviewFrameRef; + /** + * Boxes sampled along the track for the cycling animation, primary first. + * A single entry means a static detection. + */ + frames: ReviewFrameRef[]; + /** Number of keyframes in the track. */ + keyframeCount: number; + /** Type and confidence of the pair the query matched on (top pair otherwise). */ + type: string; + confidence: number; + /** Attribute mode: what matched. */ + matchedAttribute?: { + key: string; + value: unknown; + scope: 'track' | 'detection'; + }; +} + +/** + * One grid entry: a track shown once per camera it appears in. Single + * camera datasets have one item per entry; the cameras of a multicamera + * dataset share the entry, with their items' frames aligned. + */ +export interface ReviewEntry { + key: string; + items: ReviewItem[]; + /** Camera name per item; empty strings for single-camera entries. */ + labels: string[]; +} + +/** Grid presentation settings, persisted per browser. */ +export interface ReviewGridSettings { + columns: number; + rows: number; + /** + * Context around the box as a fraction of its longer side: 0.3 shows the + * box plus 30% of its size on each side. + */ + padding: number; + /** Milliseconds between frames of a cycling track. */ + cycleIntervalMs: number; + /** Most frames sampled along a track. */ + maxSequenceFrames: number; +} + +export const DEFAULT_REVIEW_QUERY: ReviewQuery = { + mode: 'type', + type: '', + threshold: 0.1, + attributeKey: '', + attributeValue: '', + attributeScope: 'any', +}; + +export const DEFAULT_REVIEW_GRID: ReviewGridSettings = { + columns: 5, + rows: 4, + padding: 0.3, + cycleIntervalMs: 400, + maxSequenceFrames: 8, +}; + +export const REVIEW_GRID_LIMITS = { + columns: [1, 12] as const, + rows: [1, 10] as const, + padding: [0, 3] as const, +}; diff --git a/client/dive-common/review/useReviewGrid.ts b/client/dive-common/review/useReviewGrid.ts new file mode 100644 index 000000000..a69105af0 --- /dev/null +++ b/client/dive-common/review/useReviewGrid.ts @@ -0,0 +1,181 @@ +/** + * Paging, zoom and chip-loading behaviour shared by every chip grid: the + * review page and any other view that shows {@link ReviewItem}s in a + * {@link ReviewGrid}. Owns the current page, keeps the chip store rendering + * at the cells' resolution and aspect ratio, and loads chips for the + * visible page (plus a prefetch of the next one). + */ +import { + computed, ref, Ref, unref, watch, +} from 'vue'; +import { debounce } from 'lodash'; +import type { ChipStore } from './chipStore'; +import { chipSizeFor } from './chipRenderer'; +import { clampGrid } from './gridSettings'; +import { REVIEW_GRID_LIMITS, ReviewGridSettings, ReviewItem } from './types'; + +const CHIP_OUTLINE = '#00e5ff'; +/** Height of a cell's footer (type field and caption), excluded from the image area. */ +const CELL_FOOTER_PX = 46; + +/** Chip aspect for a cell, coarsened so window resizes rarely force a re-render. */ +export function chipAspectFor(cellWidth: number, cellHeight: number, footerPx = CELL_FOOTER_PX): number { + const imageHeight = cellHeight - footerPx; + if (cellWidth <= 0 || imageHeight <= 0) return 1; + const ratio = Math.min(3, Math.max(1 / 3, cellWidth / imageHeight)); + return Math.round(ratio * 10) / 10; +} + +export interface ReviewGridOptions { + /** What the grid pages over: review items, or entries holding several. */ + items: Ref; + /** Items whose chips an entry needs; defaults to the entry being an item. */ + chipItemsOf?: (entry: T) => readonly ReviewItem[]; + /** Reactive grid settings (mutated in place by the setters below). */ + grid: ReviewGridSettings; + chipStore: ChipStore; + /** Whether the grid is on screen; chips only load and keys only page while true. */ + active: Ref; + /** Footer height to exclude from the chip aspect ratio. */ + footerPx?: number | Ref; + /** Box outline burned into the chips; empty draws none (the cell overlays it). */ + outline?: string; +} + +/** How long paging must be idle before chips load, so skipped pages never render. */ +const PAGE_SETTLE_MS = 250; + +export function useReviewGrid(options: ReviewGridOptions) { + const { + items, grid, chipStore, active, + } = options; + const chipItemsOf = options.chipItemsOf ?? ((entry: T) => [entry as unknown as ReviewItem]); + const chipItems = (entries: readonly T[]) => entries.flatMap((entry) => chipItemsOf(entry)); + const page = ref(0); + const cellSize = ref({ width: 0, height: 0 }); + + const perPage = computed(() => grid.columns * grid.rows); + const pageCount = computed(() => Math.max(1, Math.ceil(items.value.length / perPage.value))); + const pageItems = computed(() => items.value.slice(page.value * perPage.value, (page.value + 1) * perPage.value)); + const nextPageItems = computed(() => items.value.slice((page.value + 1) * perPage.value, (page.value + 2) * perPage.value)); + + function ensureVisible() { + if (!active.value) return; + const visible = chipItems(pageItems.value); + const prefetch = chipItems(nextPageItems.value); + chipStore.trimQueues(new Set([...visible, ...prefetch].map((i) => i.key))); + chipStore.ensurePrimary([...visible, ...prefetch]); + chipStore.ensureSequences(visible); + } + + const applyChipOptions = debounce(() => { + const pixels = Math.max(cellSize.value.width, cellSize.value.height) + * (window.devicePixelRatio || 1); + if (pixels <= 0) return; + chipStore.setOptions({ + padding: grid.padding, + size: chipSizeFor(pixels), + aspect: chipAspectFor(cellSize.value.width, cellSize.value.height, unref(options.footerPx)), + outline: options.outline ?? CHIP_OUTLINE, + }); + ensureVisible(); + }, 200); + + /** + * Rapid paging only loads the page landed on: queued work for pages + * passed over is dropped at once, and new work waits for paging to settle. + */ + const ensureVisibleSettled = debounce(ensureVisible, PAGE_SETTLE_MS); + function onPageChanged() { + if (!active.value) return; + chipStore.trimQueues(new Set(chipItems(pageItems.value).map((i) => i.key))); + ensureVisibleSettled(); + } + + watch(() => [grid.padding, cellSize.value.width, cellSize.value.height, unref(options.footerPx)], applyChipOptions); + watch(page, onPageChanged); + watch(active, ensureVisible); + watch(items, () => { + page.value = 0; + ensureVisible(); + }); + watch(perPage, () => { + page.value = Math.min(page.value, pageCount.value - 1); + ensureVisible(); + }); + + function goToPage(next: number) { + page.value = Math.min(Math.max(0, next), pageCount.value - 1); + } + + function setColumns(value: number) { + Object.assign(grid, clampGrid({ ...grid, columns: Number(value) })); + } + + function setRows(value: number) { + Object.assign(grid, clampGrid({ ...grid, rows: Number(value) })); + } + + function setPadding(value: number) { + Object.assign(grid, clampGrid({ ...grid, padding: Number(value) })); + } + + /** Fewer, larger cells (-1) or more, smaller cells (+1), keeping the shape. */ + function zoom(direction: 1 | -1) { + const ratio = grid.rows / grid.columns; + const columns = grid.columns + direction; + const rows = Math.max(1, Math.round(columns * ratio)); + Object.assign(grid, clampGrid({ ...grid, columns, rows })); + } + + const canZoomIn = computed(() => grid.columns > REVIEW_GRID_LIMITS.columns[0]); + const canZoomOut = computed(() => grid.columns < REVIEW_GRID_LIMITS.columns[1]); + + function isTypingTarget(target: EventTarget | null) { + if (!(target instanceof HTMLElement)) return false; + return ['INPUT', 'TEXTAREA', 'SELECT'].includes(target.tagName) || target.isContentEditable; + } + + /** Arrow / page keys page the grid; returns true when the key was consumed. */ + function handleKeydown(event: KeyboardEvent): boolean { + if (!active.value || isTypingTarget(event.target)) return false; + if (event.key === 'ArrowLeft' || event.key === 'PageUp') { + goToPage(page.value - 1); + } else if (event.key === 'ArrowRight' || event.key === 'PageDown') { + goToPage(page.value + 1); + } else if (event.key === 'Home') { + goToPage(0); + } else if (event.key === 'End') { + goToPage(pageCount.value - 1); + } else { + return false; + } + event.preventDefault(); + return true; + } + + function dispose() { + applyChipOptions.cancel(); + ensureVisibleSettled.cancel(); + } + + return { + page, + pageCount, + perPage, + pageItems, + cellSize, + ensureVisible, + goToPage, + setColumns, + setRows, + setPadding, + zoom, + canZoomIn, + canZoomOut, + handleKeydown, + dispose, + }; +} + +export type ReviewGridController = ReturnType; diff --git a/client/dive-common/review/viewerNavigation.spec.ts b/client/dive-common/review/viewerNavigation.spec.ts new file mode 100644 index 000000000..9ccd2de8a --- /dev/null +++ b/client/dive-common/review/viewerNavigation.spec.ts @@ -0,0 +1,15 @@ +import { parseViewerFocus, reviewViewerLocation } from './viewerNavigation'; + +describe('review viewer navigation', () => { + it('round-trips a frame and track through the query', () => { + const location = reviewViewerLocation('abc', { frame: 12, trackId: 7 }); + expect(location).toEqual({ name: 'viewer', params: { id: 'abc' }, query: { frame: '12', track: '7' } }); + expect(parseViewerFocus(location.query)).toEqual({ frame: 12, trackId: 7 }); + }); + + it('ignores missing or malformed values', () => { + expect(reviewViewerLocation('abc', {}).query).toEqual({}); + expect(parseViewerFocus({ frame: 'x', track: ['3'] })).toEqual({ trackId: 3 }); + expect(parseViewerFocus({ frame: '-1' })).toEqual({}); + }); +}); diff --git a/client/dive-common/review/viewerNavigation.ts b/client/dive-common/review/viewerNavigation.ts new file mode 100644 index 000000000..995da442e --- /dev/null +++ b/client/dive-common/review/viewerNavigation.ts @@ -0,0 +1,45 @@ +/** + * Deep links from the review grid into the annotation viewer: the viewer + * route plus query parameters naming the frame to seek to and the track to + * select once media is ready. + */ +export const VIEWER_FRAME_QUERY = 'frame'; +export const VIEWER_TRACK_QUERY = 'track'; + +export interface ViewerFocus { + frame?: number; + trackId?: number; +} + +export interface ReviewViewerLocation { + name: string; + params: Record; + query: Record; +} + +/** Both platforms name their single-dataset viewer route `viewer` with an `id` param. */ +export function reviewViewerLocation( + datasetId: string, + focus: ViewerFocus, +): ReviewViewerLocation { + const query: Record = {}; + if (focus.frame !== undefined) query[VIEWER_FRAME_QUERY] = String(focus.frame); + if (focus.trackId !== undefined) query[VIEWER_TRACK_QUERY] = String(focus.trackId); + return { name: 'viewer', params: { id: datasetId }, query }; +} + +function integerParam(value: unknown): number | undefined { + const text = Array.isArray(value) ? value[0] : value; + if (typeof text !== 'string' || !/^-?\d+$/.test(text)) return undefined; + return Number(text); +} + +/** Read the focus back out of a route query (unknown values are ignored). */ +export function parseViewerFocus(query: Record): ViewerFocus { + const frame = integerParam(query[VIEWER_FRAME_QUERY]); + const trackId = integerParam(query[VIEWER_TRACK_QUERY]); + const focus: ViewerFocus = {}; + if (frame !== undefined && frame >= 0) focus.frame = frame; + if (trackId !== undefined) focus.trackId = trackId; + return focus; +} diff --git a/client/dive-common/use/useReview.spec.ts b/client/dive-common/use/useReview.spec.ts new file mode 100644 index 000000000..4ede6f6cd --- /dev/null +++ b/client/dive-common/use/useReview.spec.ts @@ -0,0 +1,253 @@ +import { nextTick } from 'vue'; +import type { TrackData } from 'vue-media-annotator/track'; +import type { DatasetConfig } from 'dive-common/apispec'; +import { createReviewService, ReviewApi } from './useReview'; + +vi.mock('dive-common/review/frameSource', () => ({ + createFrameSource: vi.fn(() => ({ + frameCount: 1, + getFrame: vi.fn(), + dispose: vi.fn(), + })), +})); + +function track(id: number, pairs: [string, number][], frames: number[]): TrackData { + return { + id, + begin: Math.min(...frames), + end: Math.max(...frames), + confidencePairs: pairs, + attributes: {}, + features: frames.map((frame) => ({ + frame, keyframe: true, bounds: [0, 0, 10, 10], + })), + }; +} + +function config(id: string, overrides: Partial = {}): DatasetConfig { + return { + id, + name: `Dataset ${id}`, + type: 'image-sequence', + fps: 1, + createdAt: '', + subType: null, + multiCamMedia: null, + imageData: [{ url: 'a.jpg', filename: 'a.jpg' }], + videoUrl: undefined, + ...overrides, + } as DatasetConfig; +} + +function makeApi(tracksById: Record, overrides: Partial = {}): ReviewApi { + return { + loadConfig: vi.fn(async (id: string) => config(id)), + // Fresh copies each call, as a real platform returns them. + loadDetections: vi.fn(async (id: string) => ({ + version: 2, tracks: JSON.parse(JSON.stringify(tracksById[id] || [])), groups: [], sets: [], + })), + saveDetections: vi.fn(async () => undefined), + listScoringDatasets: vi.fn(async () => [{ id: 'a', name: 'Alpha' }, { id: 'b', name: 'Beta' }]), + ...overrides, + }; +} + +describe('createReviewService', () => { + it('keeps deferred datasets queued until loadQueued', async () => { + const api = makeApi({ a: [track(1, [['fish', 0.9]], [0])] }); + const service = createReviewService({ api }); + await service.addDataset('a', { id: 'a', name: 'Alpha' }, { defer: true }); + expect(service.datasets.value.map((d) => d.status)).toEqual(['queued']); + expect(api.loadDetections).not.toHaveBeenCalled(); + await service.loadQueued(); + expect(service.datasets.value.map((d) => [d.status, d.trackCount])).toEqual([['ready', 1]]); + expect(api.loadDetections).toHaveBeenCalledTimes(1); + }); + + it('loads datasets, prefers peekConfig, and builds items for a query', async () => { + const peekConfig = vi.fn(async (id: string) => config(id)); + const api = makeApi({ + a: [track(1, [['fish', 0.9]], [0, 1, 2]), track(2, [['shark', 0.3]], [4])], + }, { peekConfig }); + const service = createReviewService({ api }); + await service.refreshAvailable(); + await service.addDatasets(['a', 'a']); + expect(peekConfig).toHaveBeenCalledTimes(1); + expect(api.loadConfig).not.toHaveBeenCalled(); + expect(service.datasets.value).toHaveLength(1); + expect(service.datasets.value[0]).toMatchObject({ + id: 'a', name: 'Alpha', status: 'ready', trackCount: 2, croppable: true, + }); + expect(service.types.value).toEqual(['fish', 'shark']); + expect(service.items.value.map((i) => i.key)).toEqual(['a#1', 'a#2']); + + service.query.threshold = 0.5; + service.runQuery(); + expect(service.stale.value).toBe(false); + expect(service.items.value.map((i) => i.key)).toEqual(['a#1']); + expect(service.items.value[0].frames).toHaveLength(3); + }); + + it('expands a multicamera parent into its cameras', async () => { + const api = makeApi({ 'm/left': [track(1, [['fish', 1]], [0])], 'm/right': [] }, { + loadConfig: vi.fn(async (id: string) => (id === 'm' + ? config(id, { + type: 'multi', + multiCamMedia: { + defaultDisplay: 'left', + cameras: { + left: { type: 'image-sequence', imageData: [], videoUrl: '' }, + right: { type: 'image-sequence', imageData: [], videoUrl: '' }, + }, + }, + }) + : config(id))), + }); + const service = createReviewService({ api }); + await service.addDataset('m', { id: 'm', name: 'Rig' }); + expect(service.datasets.value.map((d) => [d.id, d.name, d.status])).toEqual([ + ['m/left', 'Rig (left)', 'ready'], + ['m/right', 'Rig (right)', 'ready'], + ]); + }); + + it('reassigns and accepts types, tracks pending edits, and saves them', async () => { + const api = makeApi({ + a: [track(1, [['fish', 0.6], ['shark', 0.4]], [0]), track(2, [['fish', 0.9]], [0])], + }); + const service = createReviewService({ api }); + await service.addDataset('a'); + service.query.type = 'fish'; + service.query.threshold = 0; + service.runQuery(); + const first = service.items.value.find((i) => i.trackId === 1)!; + const second = service.items.value.find((i) => i.trackId === 2)!; + + service.assignType(first, 'shark'); + expect(service.currentType(first)).toEqual({ type: 'shark', confidence: 1 }); + expect(service.trackOf('a', 1)?.confidencePairs).toEqual([['shark', 1]]); + expect(service.isPending(first)).toBe(true); + expect(service.pendingCount.value).toBe(1); + // The grid keeps the item until the query is run again. + expect(service.items.value).toHaveLength(2); + + service.acceptType(second); + expect(service.trackOf('a', 2)?.confidencePairs).toEqual([['fish', 1]]); + expect(service.pendingCount.value).toBe(2); + + await service.save(); + expect(api.saveDetections).toHaveBeenCalledTimes(1); + const [datasetId, args] = (api.saveDetections as ReturnType).mock.calls[0]; + expect(datasetId).toBe('a'); + expect(args.tracks.upsert.map((t: TrackData) => t.id).sort()).toEqual([1, 2]); + expect(args.groups).toEqual({ upsert: [], delete: [] }); + expect(service.pendingCount.value).toBe(0); + expect(service.error.value).toBeNull(); + }); + + it('edits a keyframe box and points, marks the track pending, and refreshes the item', async () => { + const api = makeApi({ a: [track(1, [['fish', 0.6]], [0, 4])] }); + const service = createReviewService({ api }); + await service.addDataset('a'); + service.query.type = ''; + service.query.threshold = 0; + service.runQuery(); + const [item] = service.items.value; + expect(item.primary.bounds).toEqual([0, 0, 10, 10]); + + service.updateGeometry(item, 4, { bounds: [22.4, 8, 2.6, 30], head: [5, 5], tail: [9, 9] }); + const feature = service.trackOf('a', 1)?.features.find((f) => f.frame === 4); + expect(feature?.bounds).toEqual([3, 8, 22, 30]); + expect(feature?.head).toEqual([5, 5]); + const keys = feature?.geometry?.features.map((g) => (g.properties as { key: string }).key).sort(); + expect(keys).toEqual(['HeadTails', 'head', 'tail']); + expect(item.frames.find((f) => f.frame === 4)).toMatchObject({ bounds: [3, 8, 22, 30], head: [5, 5], tail: [9, 9] }); + expect(item.primary.bounds).toEqual([0, 0, 10, 10]); + expect(service.isPending(item)).toBe(true); + + service.updateGeometry(item, 4, { tail: null }); + expect(feature?.tail).toBeUndefined(); + expect(feature?.geometry?.features.map((g) => (g.properties as { key: string }).key)).toEqual(['head']); + }); + + it('groups a track across cameras, adds boxes where a side is missing, and deletes tracks', async () => { + const api = makeApi({ 'm/left': [track(3, [['fish', 1]], [0, 4])], 'm/right': [track(3, [['fish', 1]], [4])] }, { + loadConfig: vi.fn(async (id: string) => (id === 'm' + ? config(id, { + type: 'multi', + multiCamMedia: { + defaultDisplay: 'left', + cameras: { + left: { type: 'image-sequence', imageData: [], videoUrl: '' }, + right: { type: 'image-sequence', imageData: [], videoUrl: '' }, + }, + }, + }) + : config(id))), + }); + const service = createReviewService({ api }); + await service.addDataset('m', { id: 'm', name: 'Rig' }); + service.query.threshold = 0; + service.runQuery(); + + expect(service.entries.value).toHaveLength(1); + const [entry] = service.entries.value; + expect(entry.labels).toEqual(['left', 'right']); + expect(service.parentOf('m/right')).toBe('m'); + const right = entry.items[1]; + expect(right.frames.map((f) => f.missing ?? false)).toEqual([true, false]); + + service.addKeyframe(right, 0, right.frames[0].bounds); + expect(service.trackOf('m/right', 3)?.features.map((f) => f.frame)).toEqual([0, 4]); + expect(service.trackOf('m/right', 3)?.begin).toBe(0); + expect(service.entries.value[0].items[1].frames.every((f) => !f.missing)).toBe(true); + expect(service.pendingCount.value).toBe(1); + + service.deleteTrack(entry.items[0]); + service.deleteTrack(entry.items[1]); + expect(service.entries.value).toHaveLength(0); + expect(service.pendingCount.value).toBe(2); + await service.save(); + const { calls } = (api.saveDetections as ReturnType).mock; + expect(calls.map(([id, args]) => [id, args.tracks.delete])).toEqual([['m/left', [3]], ['m/right', [3]]]); + expect(service.pendingCount.value).toBe(0); + }); + + it('reports a failed save and keeps the edits pending', async () => { + const api = makeApi({ a: [track(1, [['fish', 0.6]], [0])] }, { + saveDetections: vi.fn(async () => { throw new Error('disk full'); }), + }); + const service = createReviewService({ api }); + await service.addDataset('a'); + service.runQuery(); + service.assignType(service.items.value[0], 'shark'); + await service.save(); + expect(service.error.value).toBe('disk full'); + expect(service.pendingCount.value).toBe(1); + }); + + it('discards edits by reloading the changed datasets', async () => { + const api = makeApi({ a: [track(1, [['fish', 0.6]], [0])] }); + const service = createReviewService({ api }); + await service.addDataset('a'); + service.runQuery(); + service.assignType(service.items.value[0], 'shark'); + await service.discardChanges(); + expect(api.loadDetections).toHaveBeenCalledTimes(2); + expect(service.pendingCount.value).toBe(0); + expect(service.trackOf('a', 1)?.confidencePairs).toEqual([['fish', 0.6]]); + }); + + it('marks a dataset that failed to load and lets it be removed', async () => { + const api = makeApi({}, { + loadDetections: vi.fn(async () => { throw new Error('missing'); }), + }); + const service = createReviewService({ api }); + await service.addDataset('a'); + expect(service.datasets.value[0]).toMatchObject({ status: 'error', error: 'missing' }); + service.removeDataset('a'); + await nextTick(); + expect(service.datasets.value).toHaveLength(0); + expect(service.types.value).toEqual([]); + }); +}); diff --git a/client/dive-common/use/useReview.ts b/client/dive-common/use/useReview.ts new file mode 100644 index 000000000..ad466b24c --- /dev/null +++ b/client/dive-common/use/useReview.ts @@ -0,0 +1,681 @@ +/** + * State behind the Review page: the datasets under review (with their + * tracks held in memory), the query, the grid settings, the rendered + * chips, and the type edits waiting to be saved back. + */ +import { + computed, inject, provide, reactive, ref, Ref, watch, +} from 'vue'; +import { debounce } from 'lodash'; +import type { Api, DatasetConfig } from 'dive-common/apispec'; +import type { ScoringDatasetSummary } from 'dive-common/scoring/types'; +import type { Feature, TrackData } from 'vue-media-annotator/track'; +import type { AnnotationId, ConfidencePair } from 'vue-media-annotator/BaseAnnotation'; +import type { RectBounds } from 'vue-media-annotator/utils'; +import StyleManager from 'vue-media-annotator/StyleManager'; +import { + acceptPairAsCorrect, compileHierarchy, reassignPairs, TypeHierarchyIndex, +} from 'dive-common/typeHierarchy'; +import { createFrameSource, FrameSource } from 'dive-common/review/frameSource'; +import { createChipStore, ChipStore } from 'dive-common/review/chipStore'; +import { + buildReviewItems, CameraMembership, collectAttributeKeys, collectTypes, frameRefFor, groupReviewItems, + sortReviewItems, +} from 'dive-common/review/reviewItems'; +import { usePersistentGridSettings } from 'dive-common/review/gridSettings'; +import { + DEFAULT_REVIEW_QUERY, + ReviewEntry, + ReviewGridSettings, + ReviewItem, + ReviewPolygon, + ReviewQuery, + ReviewSortOrder, +} from 'dive-common/review/types'; + +const CHIP_OUTLINE = '#00e5ff'; + +/** Geometry edits for one keyframe; omitted fields are left alone, null removes a point. */ +export interface ReviewGeometryEdit { + bounds?: RectBounds; + polygons?: ReviewPolygon[]; + head?: [number, number] | null; + tail?: [number, number] | null; +} + +export type ReviewApi = Pick; + +export interface ReviewServiceDeps { + api: ReviewApi; +} + +/** `queued` datasets were picked but load only once results are wanted. */ +export type ReviewDatasetStatus = 'queued' | 'loading' | 'ready' | 'error'; + +export interface ReviewDataset { + id: string; + name: string; + type?: string; + status: ReviewDatasetStatus; + error?: string; + trackCount: number; + /** True when the media can be cropped into chips. */ + croppable: boolean; +} + +export interface ReviewService { + datasets: Readonly>; + available: Readonly>; + query: ReviewQuery; + grid: ReviewGridSettings; + sort: Ref; + items: Readonly>; + /** Items grouped into grid entries (one per track across its cameras). */ + entries: Readonly>; + /** Bumps whenever tracks load or change; computeds that read tracks depend on it. */ + dataRevision: Readonly>; + /** True once tracks or the query changed after the last run. */ + stale: Readonly>; + types: Readonly>; + attributeKeys: Readonly>; + pendingCount: Readonly>; + saving: Readonly>; + loading: Readonly>; + error: Readonly>; + chipStore: ChipStore; + datasetName(id: string): string; + refreshAvailable(): Promise; + addDataset(id: string, summary?: ScoringDatasetSummary, options?: { defer?: boolean }): Promise; + loadQueued(): Promise; + addDatasets(ids: string[]): Promise; + removeDataset(id: string): void; + reloadDataset(id: string): Promise; + runQuery(): void; + trackOf(datasetId: string, trackId: AnnotationId): TrackData | undefined; + /** The item's live top type/confidence after any edits. */ + currentType(item: ReviewItem): { type: string; confidence: number }; + /** The colour the annotator draws a type in (custom dataset styles applied). */ + colorFor(type: string): string; + /** Frames per second the dataset is annotated at, or 0 when unknown. */ + datasetFps(id: string): number; + /** The multicamera parent a camera dataset was expanded from, or the id itself. */ + parentOf(id: string): string; + /** Add a keyframe with a box to a track, e.g. where one camera lacks a detection. */ + addKeyframe(item: ReviewItem, frame: number, bounds: RectBounds): void; + /** Remove the track behind an item; written on the next save. */ + deleteTrack(item: ReviewItem): void; + isPending(item: ReviewItem): boolean; + assignType(item: ReviewItem, type: string): void; + acceptType(item: ReviewItem): void; + /** Change a keyframe's box, polygons or head/tail points; re-renders the item's chips. */ + updateGeometry(item: ReviewItem, frame: number, edit: ReviewGeometryEdit): void; + save(): Promise; + discardChanges(): Promise; + clearError(): void; + dispose(): void; +} + +interface LoadedDataset { + config: DatasetConfig; + tracks: Map; + hierarchy: TypeHierarchyIndex; + frameSource: FrameSource | null; + pending: Set; + /** Tracks removed here and not yet deleted on the platform. */ + deleted: Set; +} + +type GeoFeature = NonNullable['features'][number]; + +function featureKey(geo: GeoFeature): unknown { + return (geo.properties as { key?: unknown } | null)?.key; +} + +/** Write a head/tail point into the keyframe the way DIVE stores it. */ +function setKeypoint(feature: Feature, key: 'head' | 'tail', point: [number, number] | null) { + const collection = feature.geometry || { type: 'FeatureCollection' as const, features: [] }; + const remaining = collection.features.filter( + (geo) => !(geo.geometry.type === 'Point' && featureKey(geo) === key), + ); + if (point) { + remaining.push({ + type: 'Feature', + properties: { key }, + geometry: { type: 'Point', coordinates: [point[0], point[1]] }, + }); + } + collection.features = remaining; + // eslint-disable-next-line no-param-reassign + feature.geometry = collection; + if (point) { + // eslint-disable-next-line no-param-reassign + feature[key] = [point[0], point[1]]; + } else { + // eslint-disable-next-line no-param-reassign + delete feature[key]; + } +} + +/** Keep the head-to-tail line in step with its end points. */ +function syncHeadTailLine(feature: Feature) { + const collection = feature.geometry; + if (!collection) return; + const head = collection.features.find((g) => g.geometry.type === 'Point' && featureKey(g) === 'head'); + const tail = collection.features.find((g) => g.geometry.type === 'Point' && featureKey(g) === 'tail'); + collection.features = collection.features.filter( + (geo) => !(geo.geometry.type === 'LineString' && featureKey(geo) === 'HeadTails'), + ); + if (head && tail && head.geometry.type === 'Point' && tail.geometry.type === 'Point') { + collection.features.push({ + type: 'Feature', + properties: { key: 'HeadTails' }, + geometry: { type: 'LineString', coordinates: [head.geometry.coordinates, tail.geometry.coordinates] }, + }); + } +} + +/** Replace the outer rings of the keyframe's polygons, in order. */ +function setPolygons(feature: Feature, polygons: ReviewPolygon[]) { + const collection = feature.geometry; + if (!collection) return; + let index = 0; + collection.features.forEach((geo) => { + if (geo.geometry.type !== 'Polygon') return; + const ring = polygons[index]; + index += 1; + if (!ring || ring.length < 3) return; + const holes = geo.geometry.coordinates.slice(1); + // eslint-disable-next-line no-param-reassign + geo.geometry.coordinates = [[...ring.map(([x, y]) => [x, y]), [ring[0][0], ring[0][1]]], ...holes]; + }); +} + +function topPair(pairs: readonly ConfidencePair[]): { type: string; confidence: number } { + const top = pairs.reduce( + (acc, pair) => (acc === null || pair[1] > acc[1] ? pair : acc), + null, + ); + return top ? { type: top[0], confidence: top[1] } : { type: '', confidence: 0 }; +} + +export function createReviewService(deps: ReviewServiceDeps): ReviewService { + const { api } = deps; + const datasets = ref([]); + const available = ref([]); + const query = reactive({ ...DEFAULT_REVIEW_QUERY }); + const grid = usePersistentGridSettings(); + const sort = ref('confidence-desc'); + const items = ref([]); + const dataRevision = ref(0); + const stale = ref(false); + const saving = ref(false); + const loading = ref(false); + const error = ref(null); + /** Tracks and media, deliberately outside Vue reactivity (they can be large). */ + const loaded = new Map(); + /** Loads still in flight, so a removal during load is honoured. */ + let loadGeneration = 0; + /** Type colours as the annotator assigns them, seeded from each dataset's custom styles. */ + const styles = new StyleManager({ markChangesPending: () => undefined }); + /** Camera datasets expanded from a multicamera parent. */ + const memberships = new Map(); + + // Query changes apply as soon as they settle; the grid only reshuffles + // for those, never for edits made in it. + const runQuerySettled = debounce(() => runQuery(), 200); + watch(query, () => { + stale.value = true; + runQuerySettled(); + }, { deep: true }); + + const chipStore = createChipStore({ + frameSourceFor: (datasetId) => loaded.get(datasetId)?.frameSource ?? null, + }, { + padding: grid.padding, size: 256, aspect: 1, outline: CHIP_OUTLINE, + }); + + function fail(reason: unknown, fallback: string) { + const message = reason instanceof Error ? reason.message : String(reason || fallback); + error.value = message || fallback; + } + + function datasetName(id: string) { + return datasets.value.find((d) => d.id === id)?.name + || available.value.find((d) => d.id === id)?.name + || id; + } + + function entry(id: string) { + return datasets.value.find((d) => d.id === id); + } + + function patch(id: string, changes: Partial) { + datasets.value = datasets.value.map((d) => (d.id === id ? { ...d, ...changes } : d)); + } + + async function refreshAvailable() { + if (!api.listScoringDatasets) return; + try { + available.value = await api.listScoringDatasets(); + } catch (err) { + fail(err, 'Could not list datasets'); + } + } + + function loadConfig(id: string) { + return api.peekConfig ? api.peekConfig(id) : api.loadConfig(id); + } + + function frameSourceFor(config: DatasetConfig): FrameSource | null { + try { + return createFrameSource(config); + } catch { + return null; + } + } + + function dropLoaded(id: string) { + const existing = loaded.get(id); + if (existing) { + existing.frameSource?.dispose(); + loaded.delete(id); + } + } + + async function load(id: string, generation: number) { + loading.value = true; + try { + const config = await loadConfig(id); + if (generation !== loadGeneration || !entry(id)) return; + if (config.type === 'multi') { + // Review the cameras of a multicamera dataset as separate sequences. + const cameras = Object.keys(config.multiCamMedia?.cameras || {}); + const parentName = entry(id)?.name || config.name; + datasets.value = datasets.value.filter((d) => d.id !== id); + cameras.forEach((camera, rank) => memberships.set(`${id}/${camera}`, { parent: id, camera, rank })); + await Promise.all(cameras.map((camera) => addDataset(`${id}/${camera}`, { + id: `${id}/${camera}`, name: `${parentName} (${camera})`, type: config.multiCamMedia?.cameras[camera]?.type, + }))); + return; + } + const detections = await api.loadDetections(id); + if (generation !== loadGeneration || !entry(id)) return; + dropLoaded(id); + const tracks = new Map(); + detections.tracks.forEach((track) => tracks.set(track.id, track)); + const frameSource = frameSourceFor(config); + if (config.customTypeStyling) { + styles.populateTypeStyles({ ...styles.customStyles.value, ...config.customTypeStyling }); + } + loaded.set(id, { + config, + tracks, + hierarchy: compileHierarchy(config.typeHierarchy || {}), + frameSource, + pending: new Set(), + deleted: new Set(), + }); + patch(id, { + status: 'ready', + error: undefined, + name: entry(id)?.name || config.name, + type: config.type, + trackCount: tracks.size, + croppable: frameSource !== null, + }); + dataRevision.value += 1; + // Newly loaded tracks join the grid without any further action. + runQuery(); + } catch (err) { + if (generation !== loadGeneration || !entry(id)) return; + patch(id, { + status: 'error', + error: err instanceof Error ? err.message : String(err), + }); + } finally { + loading.value = datasets.value.some((d) => d.status === 'loading'); + } + } + + /** + * Add a dataset; with `defer` it only joins the list and loads on the + * next `loadQueued`, so picking many datasets costs nothing until the + * results are actually wanted. + */ + async function addDataset(id: string, summary?: ScoringDatasetSummary, options: { defer?: boolean } = {}) { + if (!id || entry(id)) return; + datasets.value = [...datasets.value, { + id, + name: summary?.name || datasetName(id), + type: summary?.type, + status: options.defer ? 'queued' : 'loading', + trackCount: 0, + croppable: false, + }]; + if (options.defer) return; + await load(id, loadGeneration); + } + + /** Load every queued dataset; annotations are read and the query rerun as each arrives. */ + async function loadQueued() { + const queued = datasets.value.filter((d) => d.status === 'queued').map((d) => d.id); + queued.forEach((id) => patch(id, { status: 'loading' })); + await Promise.all(queued.map((id) => load(id, loadGeneration))); + } + + async function addDatasets(ids: string[]) { + const unique = Array.from(new Set(ids.filter(Boolean))); + await Promise.all(unique.map((id) => addDataset(id))); + } + + function removeDataset(id: string) { + datasets.value = datasets.value.filter((d) => d.id !== id); + dropLoaded(id); + loadGeneration += 1; + dataRevision.value += 1; + runQuery(); + } + + async function reloadDataset(id: string) { + if (!entry(id)) return; + patch(id, { status: 'loading', error: undefined }); + await load(id, loadGeneration); + } + + function allTracks(): TrackData[] { + const all: TrackData[] = []; + loaded.forEach((dataset) => all.push(...dataset.tracks.values())); + return all; + } + + /** Read inside a computed so it re-runs when tracks load or change. */ + function dependOnData(): number { + return dataRevision.value; + } + + const types = computed(() => { + dependOnData(); + return collectTypes(allTracks()); + }); + + const attributeKeys = computed(() => { + dependOnData(); + const keys = new Set(); + loaded.forEach((dataset) => { + collectAttributeKeys(dataset.tracks.values(), dataset.config.attributes).forEach((k) => keys.add(k)); + }); + return Array.from(keys).sort((a, b) => a.localeCompare(b)); + }); + + function runQuery() { + const order = datasets.value.map((d) => d.id); + const built: ReviewItem[] = []; + order.forEach((id) => { + const dataset = loaded.get(id); + if (dataset) { + built.push(...buildReviewItems(id, dataset.tracks.values(), { ...query }, grid.maxSequenceFrames)); + } + }); + items.value = sortReviewItems(built, sort.value, order); + stale.value = false; + } + + watch(sort, () => { + items.value = sortReviewItems(items.value, sort.value, datasets.value.map((d) => d.id)); + }); + + function trackOf(datasetId: string, trackId: AnnotationId) { + return loaded.get(datasetId)?.tracks.get(trackId); + } + + function parentOf(id: string) { + return memberships.get(id)?.parent ?? id; + } + + const entries = computed(() => { + dependOnData(); + return groupReviewItems( + items.value, + (datasetId) => memberships.get(datasetId), + (item) => trackOf(item.datasetId, item.trackId), + grid.maxSequenceFrames, + ); + }); + + function currentType(item: ReviewItem) { + const track = trackOf(item.datasetId, item.trackId); + if (!track) return { type: item.type, confidence: item.confidence }; + if (query.mode === 'type' && query.type) { + // Keep showing the queried pair while it still exists, so an edit that + // demotes it is visible as such. + const pair = track.confidencePairs.find(([type]) => type === query.type); + const top = topPair(track.confidencePairs); + if (pair && top.type === query.type) return { type: pair[0], confidence: pair[1] }; + return top; + } + return topPair(track.confidencePairs); + } + + function colorFor(type: string) { + return styles.typeStyling.value.color(type); + } + + function datasetFps(id: string) { + const fps = Number(loaded.get(id)?.config.fps); + return Number.isFinite(fps) && fps > 0 ? fps : 0; + } + + function isPending(item: ReviewItem) { + return loaded.get(item.datasetId)?.pending.has(item.trackId) ?? false; + } + + const pendingCount = computed(() => { + dependOnData(); + let count = 0; + loaded.forEach((dataset) => { count += dataset.pending.size + dataset.deleted.size; }); + return count; + }); + + function updatePairs(item: ReviewItem, update: (pairs: ConfidencePair[], hierarchy: TypeHierarchyIndex) => ConfidencePair[]) { + const dataset = loaded.get(item.datasetId); + const track = dataset?.tracks.get(item.trackId); + if (!dataset || !track) return; + const next = update(track.confidencePairs.map(([t, c]) => [t, c] as ConfidencePair), dataset.hierarchy); + track.confidencePairs = next; + dataset.pending.add(item.trackId); + dataRevision.value += 1; + } + + function assignType(item: ReviewItem, type: string) { + const trimmed = type.trim(); + if (!trimmed) return; + const current = currentType(item); + if (current.type === trimmed && current.confidence >= 1) return; + updatePairs(item, (pairs, hierarchy) => reassignPairs( + hierarchy, + pairs, + current.type || trimmed, + trimmed, + 1, + )); + } + + function acceptType(item: ReviewItem) { + const current = currentType(item); + if (!current.type) return; + updatePairs(item, (pairs, hierarchy) => acceptPairAsCorrect(hierarchy, pairs, current.type)); + } + + function deleteTrack(item: ReviewItem) { + const dataset = loaded.get(item.datasetId); + if (!dataset || !dataset.tracks.has(item.trackId)) return; + dataset.tracks.delete(item.trackId); + dataset.pending.delete(item.trackId); + dataset.deleted.add(item.trackId); + // The entry leaves the grid at once; everything else stays put. + items.value = items.value.filter( + (other) => !(other.datasetId === item.datasetId && other.trackId === item.trackId), + ); + dataRevision.value += 1; + } + + function addKeyframe(item: ReviewItem, frame: number, bounds: RectBounds) { + const dataset = loaded.get(item.datasetId); + const track = dataset?.tracks.get(item.trackId); + if (!dataset || !track || track.features.some((f) => f.frame === frame && f.bounds)) return; + const [x1, y1, x2, y2] = bounds; + const previous = [...track.features].reverse().find((f) => f.frame < frame); + const feature: Feature = { + frame, + keyframe: true, + interpolate: previous?.interpolate ?? false, + bounds: [ + Math.round(Math.min(x1, x2)), Math.round(Math.min(y1, y2)), + Math.round(Math.max(x1, x2)), Math.round(Math.max(y1, y2)), + ], + }; + track.features = [...track.features.filter((f) => f.frame !== frame), feature] + .sort((a, b) => a.frame - b.frame); + track.begin = Math.min(track.begin, frame); + track.end = Math.max(track.end, frame); + dataset.pending.add(item.trackId); + dataRevision.value += 1; + } + + function updateGeometry(item: ReviewItem, frame: number, edit: ReviewGeometryEdit) { + const dataset = loaded.get(item.datasetId); + const track = dataset?.tracks.get(item.trackId); + const feature = track?.features.find((f) => f.frame === frame); + if (!dataset || !track || !feature) return; + + if (edit.bounds) { + const [x1, y1, x2, y2] = edit.bounds; + feature.bounds = [ + Math.round(Math.min(x1, x2)), Math.round(Math.min(y1, y2)), + Math.round(Math.max(x1, x2)), Math.round(Math.max(y1, y2)), + ]; + } + if (edit.head !== undefined) setKeypoint(feature, 'head', edit.head); + if (edit.tail !== undefined) setKeypoint(feature, 'tail', edit.tail); + if (edit.head !== undefined || edit.tail !== undefined) syncHeadTailLine(feature); + if (edit.polygons) setPolygons(feature, edit.polygons); + + // The grid item mirrors the keyframe; refresh it so overlays and the + // re-rendered chip follow the edit. + const refreshed = frameRefFor(feature); + if (refreshed) { + item.frames.forEach((ref) => { + if (ref.frame === frame) Object.assign(ref, refreshed); + }); + if (item.primary.frame === frame) Object.assign(item.primary, refreshed); + } + dataset.pending.add(item.trackId); + dataRevision.value += 1; + // The chip keeps its crop: the box is drawn over it, so the view does + // not jump when an edit lands. + } + + async function save() { + if (saving.value) return; + saving.value = true; + error.value = null; + try { + const targets = Array.from(loaded.entries()) + .filter(([, d]) => d.pending.size > 0 || d.deleted.size > 0); + const results = await Promise.allSettled(targets.map(async ([id, dataset]) => { + const upsert = Array.from(dataset.pending) + .map((trackId) => dataset.tracks.get(trackId)) + .filter((t): t is TrackData => !!t); + await api.saveDetections(id, { + tracks: { upsert, delete: Array.from(dataset.deleted) }, + groups: { upsert: [], delete: [] }, + }); + dataset.pending.clear(); + dataset.deleted.clear(); + })); + const failed = results.find((r) => r.status === 'rejected') as PromiseRejectedResult | undefined; + if (failed) throw failed.reason; + } catch (err) { + fail(err, 'Could not save the changed annotations'); + } finally { + dataRevision.value += 1; + saving.value = false; + } + } + + async function discardChanges() { + const dirty = Array.from(loaded.entries()) + .filter(([, d]) => d.pending.size > 0 || d.deleted.size > 0).map(([id]) => id); + await Promise.all(dirty.map((id) => reloadDataset(id))); + } + + function clearError() { + error.value = null; + } + + function dispose() { + runQuerySettled.cancel(); + loadGeneration += 1; + loaded.forEach((dataset) => dataset.frameSource?.dispose()); + loaded.clear(); + chipStore.reset(); + } + + return { + datasets, + available, + query, + grid, + sort, + items, + entries, + dataRevision, + stale, + types, + attributeKeys, + pendingCount, + saving, + loading, + error, + chipStore, + datasetName, + refreshAvailable, + addDataset, + addDatasets, + loadQueued, + removeDataset, + reloadDataset, + runQuery, + trackOf, + currentType, + colorFor, + datasetFps, + parentOf, + addKeyframe, + deleteTrack, + isPending, + assignType, + acceptType, + updateGeometry, + save, + discardChanges, + clearError, + dispose, + }; +} + +const ReviewSymbol = Symbol('review'); + +export function provideReview(service: ReviewService) { + provide(ReviewSymbol, service); +} + +export function useReview(): ReviewService { + const service = inject(ReviewSymbol, null); + if (!service) { + throw new Error('Review service not provided'); + } + return service; +} diff --git a/client/platform/desktop/frontend/api.ts b/client/platform/desktop/frontend/api.ts index 26e14b08e..ad1a48489 100644 --- a/client/platform/desktop/frontend/api.ts +++ b/client/platform/desktop/frontend/api.ts @@ -826,6 +826,11 @@ async function loadConfig(id: string) { return { ...data, calibration: data.multiCam?.calibration ?? null }; } +/** loadConfig without the recents bookkeeping the stateful wrapper adds. */ +function peekConfig(id: string) { + return loadConfig(id); +} + let scoringAnnotationPreviewFile: string | null = null; /** One-shot annotation file to load in the viewer (from scoring result links). */ @@ -939,6 +944,7 @@ export { exportScoringPdf, /* Standard Specification APIs */ loadConfig, + peekConfig, loadDetections, loadFrameMetadata, getPipelineList, diff --git a/client/platform/desktop/frontend/components/MultiPipeline.vue b/client/platform/desktop/frontend/components/MultiPipeline.vue index 5e7399611..4ae647f81 100644 --- a/client/platform/desktop/frontend/components/MultiPipeline.vue +++ b/client/platform/desktop/frontend/components/MultiPipeline.vue @@ -11,7 +11,6 @@ import { useRoute, useRouter } from 'vue-router/composables'; import { Pipe, Pipelines, useApi } from 'dive-common/apispec'; import { parentDatasetId } from 'dive-common/compositeDatasetId'; import { - itemsPerPageOptions, stereoPipelineMarker, multiCamPipelineMarkers, MultiType, @@ -23,6 +22,7 @@ import { pipelineRequiresCalibration, } from 'dive-common/pipelineCalibration'; import PipelineCalibrationWarningIcon from 'dive-common/components/PipelineCalibrationWarningIcon.vue'; +import DatasetPicker from 'dive-common/components/DatasetPicker.vue'; import { usePrompt } from 'dive-common/vue-utilities/prompt-service'; import { clientSettings } from 'dive-common/store/settings'; import { datasets, JsonConfigCache } from '../store/dataset'; @@ -80,14 +80,6 @@ const headersTmpl: DataTableHeader[] = [ width: 80, }, ]; -const availableDatasetHeaders = headersTmpl.concat( - { - text: 'Include', - value: 'include', - sortable: false, - width: 80, - }, -); const stagedDatasetHeaders: DataTableHeader[] = headersTmpl.concat([ { text: 'Remove', @@ -113,10 +105,8 @@ function computeOutputDatasetName(item: JsonConfigCache) { const timeStamp = (new Date()).toISOString().replace(/[:.]/g, '-'); return `${selectedPipeline.value?.name}_${item.name}_${timeStamp}`; } +/** Every dataset, narrowed to stereo ones once a measurement pipeline type is chosen. */ function getAvailableItems(): JsonConfigCache[] { - if (!selectedPipelineType.value || !selectedPipeline.value) { - return []; - } if (selectedPipelineType.value === stereoPipelineMarker) { // Only allow stereo datasets to be included for bulk pipeline // operations if the selected pipeline type is a measurement. @@ -126,8 +116,7 @@ function getAvailableItems(): JsonConfigCache[] { } return Object.values(datasets.value); } -const availableItems: Ref = ref([]); -const availableDatasetSearch = ref(''); +const availableItems = computed(() => getAvailableItems()); const stagedDatasetIds: Ref = ref([]); const stagedDatasets = computed(() => availableItems.value.filter((item: JsonConfigCache) => stagedDatasetIds.value.includes(item.id))); const calibrationAvailableByDatasetId = ref>({}); @@ -170,9 +159,6 @@ function isPipelineItemDisabledForCalibration(pipe: Pipe) { ); } -watch(selectedPipeline, () => { - availableItems.value = getAvailableItems(); -}); function toggleStaged(item: JsonConfigCache) { if (stagedDatasetIds.value.includes(item.id)) { stagedDatasetIds.value = stagedDatasetIds.value.filter((id: string) => id !== item.id); @@ -180,28 +166,14 @@ function toggleStaged(item: JsonConfigCache) { stagedDatasetIds.value.push(item.id); } } -function datasetMatchesSearch(item: JsonConfigCache, search: string) { - if (!search) { - return true; - } - const record = item as unknown as Record; - return headersTmpl.some((header) => { - const value = String(record[header.value] ?? '').toLowerCase(); - return value.includes(search); - }); +/** Stage the picked datasets that are not staged yet. */ +function stageIds(ids: string[]) { + const staged = new Set(stagedDatasetIds.value); + stagedDatasetIds.value = stagedDatasetIds.value.concat(ids.filter((id) => !staged.has(id))); } -/* Mirrors the table's default search so "select all" only stages what is listed. */ -const unstagedSearchMatches = computed(() => { - const search = availableDatasetSearch.value.trim().toLowerCase(); - return availableItems.value.filter((item) => ( - !stagedDatasetIds.value.includes(item.id) - && datasetMatchesSearch(item, search) - )); -}); -function stageAllAvailable() { - stagedDatasetIds.value = stagedDatasetIds.value.concat( - unstagedSearchMatches.value.map((item) => item.id), - ); +function unstageIds(ids: string[]) { + const dropped = new Set(ids); + stagedDatasetIds.value = stagedDatasetIds.value.filter((id) => !dropped.has(id)); } async function runPipelineForDatasets() { @@ -243,7 +215,6 @@ async function runPipelineForDatasets() { onBeforeMount(async () => { stagedDatasetIds.value = preselectedDatasetIds(); unsortedPipelines.value = await getPipelineList(); - availableItems.value = getAvailableItems(); }); @@ -251,16 +222,18 @@ onBeforeMount(async () => { Score the selected datasets against ground truth + + + Review the selected datasets' annotations as a grid +