diff --git a/addOns/README.md b/addOns/README.md index 88f8868a6a..6a102d6968 100644 --- a/addOns/README.md +++ b/addOns/README.md @@ -1,3 +1,8 @@ # External Dependencies This module contains optional and external dependencies for including in OHIF, such as the DICOM Microscopy Viewer component. + + +# External Components + +This directory contains various external components. These can be fetched in various ways such as NPM dependencies or by fetching files externally. diff --git a/bun.lockb b/bun.lockb index 581f58e525..aebda1b817 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/common/reviews/api/ai.api.md b/common/reviews/api/ai.api.md new file mode 100644 index 0000000000..f7e0984b46 --- /dev/null +++ b/common/reviews/api/ai.api.md @@ -0,0 +1,184 @@ +## API Report File for "@cornerstonejs/ai" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts + +import type ColorTransferFunction from '@kitware/vtk.js/Rendering/Core/ColorTransferFunction'; +import { Corners } from '@kitware/vtk.js/Interaction/Widgets/OrientationMarkerWidget/Constants'; +import type { IColorMapPreset } from '@kitware/vtk.js/Rendering/Core/ColorTransferFunction/ColorMaps'; +import type { mat3 } from 'gl-matrix'; +import { mat4 } from 'gl-matrix'; +import { PixelDataTypedArray as PixelDataTypedArray_2 } from 'packages/core/dist/esm/types'; +import type { Range as Range_2 } from '@kitware/vtk.js/types'; +import { vec3 } from 'gl-matrix'; +import type vtkActor from '@kitware/vtk.js/Rendering/Core/Actor'; +import vtkAnnotatedCubeActor from '@kitware/vtk.js/Rendering/Core/AnnotatedCubeActor'; +import type { vtkCamera } from '@kitware/vtk.js/Rendering/Core/Camera'; +import { vtkColorTransferFunction } from '@kitware/vtk.js/Rendering/Core/ColorTransferFunction'; +import { vtkImageData } from '@kitware/vtk.js/Common/DataModel/ImageData'; +import type vtkImageSlice from '@kitware/vtk.js/Rendering/Core/ImageSlice'; +import type { vtkObject } from '@kitware/vtk.js/interfaces'; +import type vtkOpenGLTexture from '@kitware/vtk.js/Rendering/OpenGL/Texture'; +import type vtkPiecewiseFunction from '@kitware/vtk.js/Common/DataModel/PiecewiseFunction'; +import vtkPlane from '@kitware/vtk.js/Common/DataModel/Plane'; +import type vtkPolyData from '@kitware/vtk.js/Common/DataModel/PolyData'; +import type vtkRenderer from '@kitware/vtk.js/Rendering/Core/Renderer'; +import type vtkVolume from '@kitware/vtk.js/Rendering/Core/Volume'; + +// @public (undocumented) +export class ONNXSegmentationController { + constructor(options?: { + listeners: any; + getPromptAnnotations: any; + promptAnnotationTypes: any; + models: any; + modelName: any; + previewToolType: string; + }); + // (undocumented) + protected annotationModifiedListener: (_event?: any) => void; + // (undocumented) + protected annotationsNeedUpdating: boolean; + // (undocumented) + protected boxRadius: number; + // (undocumented) + cacheImageEncodings(current?: any, offset?: number, length?: number): any; + // (undocumented) + canvas: HTMLCanvasElement; + // (undocumented) + canvasMask: HTMLCanvasElement; + // (undocumented) + clear(viewport: any): void; + // (undocumented) + createLabelmap(mask: any, canvasPosition: any, _points: any, _labels: any): void; + // (undocumented) + protected currentImage: any; + // (undocumented) + decode(points: any, labels: any, useSession?: any): Promise; + // (undocumented) + protected desiredImage: { + imageId: any; + sampleImageId: any; + imageIndex: number; + decoder: any; + encoder: any; + }; + // (undocumented) + disconnectViewport(viewport: any): void; + // (undocumented) + protected excludeTool: string; + // (undocumented) + fetchAndCacheModel(url: any, name: any): Promise; + // (undocumented) + getConfig(modelName?: string): any; + // (undocumented) + getDirectoryForImageId(session: any, imageId: any): Promise; + // (undocumented) + getFileNameForImageId(imageId: any, extension: any): any; + // (undocumented) + protected getPromptAnnotations: (viewport?: any) => cornerstoneTools.Types.Annotations; + // (undocumented) + protected handleImage({ imageId, sampleImageId }: { + imageId: any; + sampleImageId: any; + }, imageSession: any): Promise; + // (undocumented) + protected imageEncodings: Map; + // (undocumented) + protected imageImageData: any; + // (undocumented) + initModel(): Promise; + // (undocumented) + initViewport(viewport: any): void; + // (undocumented) + interpolateScroll(viewport?: any, dir?: number): Promise; + // (undocumented) + protected isGpuInUse: boolean; + // (undocumented) + protected load(): Promise; + // (undocumented) + loadModels(models: any, imageSession?: any): Promise; + // (undocumented) + loadStorageImageEncoding(session: any, imageId: any, index?: any): Promise; + // (undocumented) + protected log(logger: Loggers, ...args: any[]): void; + // (undocumented) + mapAnnotationPoint(worldPoint: any): number[]; + // (undocumented) + static MarkerExclude: string; + // (undocumented) + static MarkerInclude: string; + // (undocumented) + protected maskImageData: any; + // (undocumented) + maxHeight: number; + // (undocumented) + maxWidth: number; + // (undocumented) + modelHeight: number; + // (undocumented) + static MODELS: { + sam_l: ({ + name: string; + url: string; + size: number; + key: string; + feedType: string; + } | { + name: string; + url: string; + size: number; + key: string; + feedType?: undefined; + })[]; + sam_h: ({ + name: string; + url: string; + size: number; + key: string; + feedType: string; + } | { + name: string; + url: string; + size: number; + key: string; + feedType?: undefined; + })[]; + }; + // (undocumented) + modelWidth: number; + // (undocumented) + protected previewToolType: string; + // (undocumented) + protected promptAnnotationTypes: string[]; + // (undocumented) + restoreImageEncoding(session: any, imageId: any): Promise; + // (undocumented) + protected runDecode(): Promise; + // (undocumented) + protected sharedImageEncoding: any; + // (undocumented) + storeImageEncoding(session: any, imageId: any, data: any): Promise; + // (undocumented) + protected tool: any; + // (undocumented) + tryLoad(options?: { + resetImage: boolean; + }): void; + // (undocumented) + updateAnnotations(): void; + // (undocumented) + protected viewport: any; + // (undocumented) + static viewportOptions: { + displayArea: Types.DisplayArea; + background: Types.Point3; + }; + // (undocumented) + protected viewportRenderedListener: (_event: any) => void; +} + +// (No @packageDocumentation comment for this package) + +``` diff --git a/common/reviews/api/core.api.md b/common/reviews/api/core.api.md index 4d418c10a7..3073520b02 100644 --- a/common/reviews/api/core.api.md +++ b/common/reviews/api/core.api.md @@ -4751,7 +4751,7 @@ export class VolumeViewport extends BaseVolumeViewport { // (undocumented) getCurrentImageId: () => string | undefined; // (undocumented) - getCurrentImageIdIndex: (volumeId?: string) => number; + getCurrentImageIdIndex: (volumeId?: string, useSlabThickness?: boolean) => number; // (undocumented) getCurrentSlicePixelData(): PixelDataTypedArray; // (undocumented) diff --git a/common/reviews/api/tools.api.md b/common/reviews/api/tools.api.md index d9a40a2f41..a75a559ea6 100644 --- a/common/reviews/api/tools.api.md +++ b/common/reviews/api/tools.api.md @@ -241,6 +241,7 @@ export class AngleTool extends AnnotationTool { type Annotation = { annotationUID?: string; parentAnnotationUID?: string; + interpolationUID?: string; childAnnotationUIDs?: string[]; highlighted?: boolean; isLocked?: boolean; @@ -270,7 +271,7 @@ type Annotation = { [key: string]: unknown; }; [key: string]: unknown; - cachedStats?: unknown; + cachedStats?: Record; }; }; @@ -308,6 +309,8 @@ type AnnotationCompletedEventType = Types_2.CustomEventType void; }; // (undocumented) + static defaults: { + configuration: { + strategies: {}; + defaultStrategy: any; + activeStrategy: any; + strategyOptions: {}; + }; + }; + // (undocumented) protected getTargetId(viewport: Types_2.IViewport): string | undefined; // (undocumented) protected getTargetImageData(targetId: string): Types_2.IImageData | Types_2.CPUIImageData; @@ -663,6 +675,8 @@ export abstract class BaseTool { // (undocumented) protected memo: utilities_2.HistoryMemo.Memo; // (undocumented) + static mergeDefaultProps(defaultProps?: {}, additionalProps?: any): any; + // (undocumented) mode: ToolModes; // (undocumented) redo(): void; @@ -677,6 +691,8 @@ export abstract class BaseTool { // (undocumented) static toolName: any; // (undocumented) + get toolName(): string; + // (undocumented) undo(): void; } @@ -821,103 +837,9 @@ declare namespace boundingBox { type BoundsIJK_2 = Types_2.BoundsIJK; // @public (undocumented) -export class BrushTool extends BaseTool { +export class BrushTool extends LabelmapBaseTool { constructor(toolProps?: PublicToolProps, defaultToolProps?: ToolProps); // (undocumented) - acceptPreview(element?: HTMLDivElement): void; - // (undocumented) - createEditData(element: any): { - volumeId: string; - referencedVolumeId: any; - segmentsLocked: number[] | []; - imageId?: undefined; - override?: undefined; - } | { - imageId: string; - segmentsLocked: number[] | []; - override: { - voxelManager: Types_2.IVoxelManager | Types_2.IVoxelManager; - imageData: vtkImageData; - }; - volumeId?: undefined; - referencedVolumeId?: undefined; - } | { - imageId: string; - segmentsLocked: number[] | []; - volumeId?: undefined; - referencedVolumeId?: undefined; - override?: undefined; - }; - // (undocumented) - protected getOperationData(element?: any): { - points: any; - segmentIndex: number; - previewColors: any; - viewPlaneNormal: any; - toolGroupId: string; - segmentationId: string; - viewUp: any; - strategySpecificConfiguration: any; - preview: unknown; - override: { - voxelManager: Types_2.IVoxelManager; - imageData: vtkImageData; - }; - segmentsLocked: number[]; - imageId?: string; - imageIds?: string[]; - volumeId?: string; - referencedVolumeId?: string; - } | { - points: any; - segmentIndex: number; - previewColors: any; - viewPlaneNormal: any; - toolGroupId: string; - segmentationId: string; - viewUp: any; - strategySpecificConfiguration: any; - preview: unknown; - volumeId: string; - referencedVolumeId: any; - segmentsLocked: number[] | []; - imageId?: undefined; - override?: undefined; - } | { - points: any; - segmentIndex: number; - previewColors: any; - viewPlaneNormal: any; - toolGroupId: string; - segmentationId: string; - viewUp: any; - strategySpecificConfiguration: any; - preview: unknown; - imageId: string; - segmentsLocked: number[] | []; - override: { - voxelManager: Types_2.IVoxelManager | Types_2.IVoxelManager; - imageData: vtkImageData; - }; - volumeId?: undefined; - referencedVolumeId?: undefined; - } | { - points: any; - segmentIndex: number; - previewColors: any; - viewPlaneNormal: any; - toolGroupId: string; - segmentationId: string; - viewUp: any; - strategySpecificConfiguration: any; - preview: unknown; - imageId: string; - segmentsLocked: number[] | []; - volumeId?: undefined; - referencedVolumeId?: undefined; - override?: undefined; - }; - // (undocumented) getStatistics(element: any, segmentIndices?: any): any; // (undocumented) invalidateBrushCursor(): void; @@ -934,7 +856,7 @@ export class BrushTool extends BaseTool { // (undocumented) previewCallback: () => void; // (undocumented) - rejectPreview(element?: HTMLDivElement): void; + prg: any; // (undocumented) renderAnnotation(enabledElement: Types_2.IEnabledElement, svgDrawingHelper: SVGDrawingHelper): void; // (undocumented) @@ -3904,7 +3826,7 @@ export class PlanarFreehandROITool extends ContourSegmentationBaseTool { // (undocumented) cancel: (element: HTMLDivElement) => void; // (undocumented) - protected createAnnotation(evt: EventTypes_2.InteractionEventType): Annotation; + protected createAnnotation(evt: EventTypes_2.InteractionEventType): ContourAnnotation; // (undocumented) filterInteractableAnnotationsForElement(element: HTMLDivElement, annotations: Annotations): Annotations | undefined; // (undocumented) @@ -4093,13 +4015,13 @@ interface ProbeAnnotation extends Annotation { // @public (undocumented) export class ProbeTool extends AnnotationTool { - constructor(toolProps?: PublicToolProps, defaultToolProps?: ToolProps); + constructor(toolProps?: PublicToolProps, defaultToolProps?: any); // (undocumented) _activateModify: (element: any) => void; // (undocumented) addNewAnnotation: (evt: EventTypes_2.InteractionEventType) => ProbeAnnotation; // (undocumented) - _calculateCachedStats(annotation: any, renderingEngine: any, enabledElement: any): any; + _calculateCachedStats(annotation: any, renderingEngine: any, enabledElement: any, changeType?: ChangeTypes): any; // (undocumented) cancel: (element: HTMLDivElement) => string; // (undocumented) @@ -4134,6 +4056,15 @@ export class ProbeTool extends AnnotationTool { // (undocumented) isPointNearTool(): boolean; // (undocumented) + static probeDefaults: { + supportedInteractionTypes: string[]; + configuration: { + shadow: boolean; + preventHandleOutsideImage: boolean; + getTextLines: typeof defaultGetTextLines; + }; + }; + // (undocumented) renderAnnotation: (enabledElement: Types_2.IEnabledElement, svgDrawingHelper: SVGDrawingHelper) => boolean; // (undocumented) static toolName: any; @@ -5215,7 +5146,7 @@ export class SplineROITool extends ContourSegmentationBaseTool { // (undocumented) cancel(element: HTMLDivElement): string; // (undocumented) - protected createAnnotation(evt: EventTypes_2.InteractionEventType): Annotation; + protected createAnnotation(evt: EventTypes_2.InteractionEventType): ContourAnnotation; // (undocumented) protected createInterpolatedSplineControl(annotation: any): void; // (undocumented) @@ -5380,6 +5311,8 @@ enum StrategyCallbacks { // (undocumented) AcceptPreview = "acceptPreview", // (undocumented) + AddPreview = "addPreview", + // (undocumented) ComputeInnerCircleRadius = "computeInnerCircleRadius", // (undocumented) CreateIsInThreshold = "createIsInThreshold", diff --git a/package.json b/package.json index a6bb9a4419..d1a12d8d5c 100644 --- a/package.json +++ b/package.json @@ -19,11 +19,12 @@ "build": "npx lerna run build --stream && npx lerna run build:loader", "build:esm": "npx lerna run build:esm --stream", "watch": "npx lerna watch -- lerna run build --scope=$LERNA_PACKAGE_NAME --include-dependents", + "build:update-api:ai": "cd packages/ai && npm run build:update-api", "build:update-api:core": "cd packages/core && npm run build:update-api", "build:update-api:tools": "cd packages/tools && npm run build:update-api", "build:update-api:nifti": "cd packages/nifti-volume-loader && npm run build:update-api", "build:update-api:dicomImageLoader": "cd packages/dicomImageLoader && npm run build:update-api", - "build:update-api": "npm run build:update-api:core && npm run build:update-api:tools && npm run build:update-api:nifti && npm run build:update-api:dicomImageLoader", + "build:update-api": "npm run build:update-api:ai && npm run build:update-api:core && npm run build:update-api:tools && npm run build:update-api:nifti && npm run build:update-api:dicomImageLoader", "clean": "npx lerna run clean --stream", "clean:deep": "npx lerna run clean:deep --stream", "example": "node ./utils/ExampleRunner/example-runner-cli.js", diff --git a/packages/adapters/examples/segmentationVolume/index.ts b/packages/adapters/examples/segmentationVolume/index.ts index 515d3a6e2a..a8ec4a7916 100644 --- a/packages/adapters/examples/segmentationVolume/index.ts +++ b/packages/adapters/examples/segmentationVolume/index.ts @@ -282,7 +282,6 @@ async function loadSegmentation(arrayBuffer: ArrayBuffer) { // const derivedVolume = await addSegmentationsToState(newSegmentationId); - derivedVolume?.voxelManager?.setCompleteScalarDataArray?.( new Uint8Array(generateToolState.labelmapBufferArray[0]) ); diff --git a/packages/ai/README.md b/packages/ai/README.md new file mode 100644 index 0000000000..7b3d075576 --- /dev/null +++ b/packages/ai/README.md @@ -0,0 +1,93 @@ +# Cornerstone AI Client Package + +This package provides AI interfaces for use with Cornerstone in client-side applications. It is designed to support the use of ONNX models, ensuring a clean separation between server-side AI processing and client-specific functionalities tied to Cornerstone. + +## Key Features + +- **ONNX Runtime Web Integration**: The package leverages the ONNX Runtime Web library, enabling AI models to run directly in the browser without relying on server-side execution. +- **Initial Model - Segment Anything Model (SAM)**: Our first supported model is the Segment Anything Model (SAM) https://segment-anything.com/, designed for segmentation tasks. + +## Getting Started + +### Running the Example + +To see the package in action with the Segment Anything Model, use the following command: + +```bash +yarn run example segmentAnythingClientSide +``` + +This will load the SAM model in the browser and allow you to perform segmentation tasks on images. + +### Model Files + +The package does not include model binaries due to their size and to give users the flexibility to use their own models. You can download pre-trained model binaries from the following links: + +Base model (vit_b) - 178 MB compressed + +- https://ohif-assets-new.s3.us-east-1.amazonaws.com/SAM/sam_b.zip + +Large model (vit_l) - 1.16 GB compressed + +- https://ohif-assets-new.s3.us-east-1.amazonaws.com/SAM/sam_l.zip + +Huge model (vit_h) - 2.38 GB compressed + +- https://ohif-assets-new.s3.us-east-1.amazonaws.com/SAM/sam_h.zip + +For the examples we are using the model url and fetch it from the web. If you see in example code we have: + + +#### URL to the model files + +```js +const models = { + sam_b: [ + { + name: 'sam-b-encoder', + url: 'https://huggingface.co/schmuell/sam-b-fp16/resolve/main/sam_vit_b_01ec64.encoder-fp16.onnx', + size: 180, + key: 'encoder', + }, + { + name: 'sam-b-decoder', + url: 'https://huggingface.co/schmuell/sam-b-fp16/resolve/main/sam_vit_b_01ec64.decoder.onnx', + size: 17, + key: 'decoder', + }, + ], +}; + +const ai = new ONNXSegmentationController({ + listeners: [mlLogger], + models, + modelName: 'sam_b', +}); +``` + +which gives the url to the model files. + +#### Models in binary + +You can download the model files and use them offline by moving them to the public folder. + +In Webpack you can add the model files to the public folder like this + +```js +new CopyPlugin({ + patterns: [ + { + from: + '../../../externals/sam_l', + to: '${destPath.replace(/\\/g, '/')}/sam_l', + }, + { + from: + '../../../externals/sam_h', + to: '${destPath.replace(/\\/g, '/')}/sam_h', + }, + ], +}), +``` + +, other build systems might have a different way to do this. diff --git a/packages/ai/api-extractor.json b/packages/ai/api-extractor.json new file mode 100644 index 0000000000..4ddb5f44d3 --- /dev/null +++ b/packages/ai/api-extractor.json @@ -0,0 +1,9 @@ +{ + "extends": "../../api-extractor.json", + "projectFolder": ".", + "mainEntryPointFilePath": "/dist/esm/index.d.ts", + "apiReport": { + "reportFileName": ".api.md", + "reportFolder": "../../common/reviews/api" + } +} diff --git a/packages/ai/examples/segmentAnthingClientSide/index.ts b/packages/ai/examples/segmentAnthingClientSide/index.ts new file mode 100644 index 0000000000..6ca5fe9bc7 --- /dev/null +++ b/packages/ai/examples/segmentAnthingClientSide/index.ts @@ -0,0 +1,409 @@ +import type { Types } from '@cornerstonejs/core'; +import { + Enums, + RenderingEngine, + imageLoader, + setVolumesForViewports, + volumeLoader, +} from '@cornerstonejs/core'; +import * as cornerstoneTools from '@cornerstonejs/tools'; +import { + addButtonToToolbar, + addDropdownToToolbar, + createImageIdsAndCacheMetaData, + initDemo, + setTitleAndDescription, + addManipulationBindings, + getLocalUrl, + addSegmentIndexDropdown, + labelmapTools, + annotationTools, +} from '../../../../utils/demo/helpers'; + +import { ONNXSegmentationController } from '@cornerstonejs/ai'; + +// This is for debugging purposes +console.warn( + 'Click on index.ts to open source code for this example --------->' +); + +const { + ToolGroupManager, + Enums: csToolsEnums, + segmentation, +} = cornerstoneTools; + +const configuration = { + preview: { + enabled: true, + previewColors: { + 0: [255, 255, 255, 128], + 1: [0, 255, 255, 255], + }, + }, +}; +const logs = []; + +/** + * Log to the specified logger. + */ +function mlLogger(logName, ...args) { + console.log(logName, ...args); + const element = document.getElementById(logName); + if (!element) { + return; + } + if (logName === 'status') { + logs.push(args.join(' ')); + if (logs.length > 5) { + logs.splice(0, 1); + } + element.innerText = logs.join('\n'); + return; + } + element.innerText = args.join(' '); +} + +const models = { + sam_b: [ + { + name: 'sam-b-encoder', + url: 'https://huggingface.co/schmuell/sam-b-fp16/resolve/main/sam_vit_b_01ec64.encoder-fp16.onnx', + size: 180, + key: 'encoder', + }, + { + name: 'sam-b-decoder', + url: 'https://huggingface.co/schmuell/sam-b-fp16/resolve/main/sam_vit_b_01ec64.decoder.onnx', + size: 17, + key: 'decoder', + }, + ], +}; + +const ai = new ONNXSegmentationController({ + listeners: [mlLogger], + models, + modelName: 'sam_b', +}); + +const { ViewportType, Events } = Enums; +const { KeyboardBindings, MouseBindings } = csToolsEnums; +const { style: toolStyle } = cornerstoneTools.annotation.config; +const volumeId = 'volumeId'; + +// Define various constants for the tool definition +const toolGroupId = 'DEFAULT_TOOLGROUP_ID'; +const volumeToolGroupId = 'VOLUME_TOOLGROUP_ID'; + +const segmentationId = `SEGMENTATION_ID`; +// Stores information on whether the AI data is encoded in cache +let cached; +let toolForPreview; + +const toolMap = new Map(annotationTools); +const defaultTool = ONNXSegmentationController.MarkerInclude; +toolMap.set(defaultTool, { + baseTool: cornerstoneTools.ProbeTool.toolName, + configuration: { + ...configuration, + getTextLines: () => null, + }, +}); +toolStyle.getDefaultToolStyles()[defaultTool] = { color: 'blue' }; + +const excludeTool = ONNXSegmentationController.MarkerExclude; +toolMap.set(excludeTool, { + baseTool: cornerstoneTools.ProbeTool.toolName, + bindings: [{ mouseButton: MouseBindings.Secondary }], + configuration: { + ...configuration, + getTextLines: () => null, + }, +}); +toolStyle.getDefaultToolStyles()[excludeTool] = { + color: 'pink', + colorSelected: 'red', +}; + +for (const [key, value] of labelmapTools.toolMap) { + toolMap.set(key, value); +} + +toolMap.set(cornerstoneTools.ZoomTool.toolName, { + bindings: [ + { + mouseButton: MouseBindings.Auxiliary, + modifierKey: KeyboardBindings.Ctrl, + }, + ], +}); + +// ======== Set up page ======== // + +setTitleAndDescription( + 'Segmentation AI', + 'Here we demonstrate how to use various predictive AI/ML techniques to aid your segmentation. ' + + 'The default model here uses "MarkerInclude" and "MarkerExclude" as segmentation AI prompts ' + + 'for the Segment Anything Model to use to generate a segmentation of the area of interest. ' + + 'Then, these prompts can be copied to the next image by pressing the "n" key to interpolate ' + + 'markers on the current slice onto the next slice.' +); + +const { canvas, canvasMask } = ai; + +const size = `24vw`; +const content = document.getElementById('content'); + +addButtonToToolbar({ + title: 'Clear', + onClick: () => { + ai.clear(activeViewport); + viewport.render(); + }, +}); + +const viewportGrid = document.createElement('div'); +let renderingEngine; +let viewport, volumeViewport, activeViewport; + +viewportGrid.style.width = '99vw'; + +const viewportId = 'Stack'; +const viewportIds = [viewportId, 'AXIAL', 'SAGITAL', 'CORONAL']; + +const elements = []; +for (const id of viewportIds) { + const el = document.createElement('div'); + elements.push(el); + el.oncontextmenu = () => false; + el.id = id; + + Object.assign(el.style, { + width: size, + height: size, + display: 'inline-block', + }); + viewportGrid.appendChild(el); +} +const [element0, element1, element2, element3] = elements; +// Uncomment these to show the canvas/mask overlays separately +// viewportGrid.appendChild(canvas); +// viewportGrid.appendChild(canvasMask); + +Object.assign(canvas.style, { + width: size, + height: size, + display: 'inline-block', + background: 'red', +}); + +Object.assign(canvasMask.style, { + width: size, + height: size, + display: 'inline-block', + background: 'black', +}); + +// viewportGrid.appendChild(canvas); +// viewportGrid.appendChild(canvasMask); + +content.appendChild(viewportGrid); + +const encoderLatency = document.createElement('div'); +encoderLatency.id = 'encoder'; +content.appendChild(encoderLatency); + +const decoderLatency = document.createElement('div'); +decoderLatency.id = 'decoder'; +content.appendChild(decoderLatency); + +const logDiv = document.createElement('div'); +logDiv.id = 'status'; +content.appendChild(logDiv); + +// ============================= // +addDropdownToToolbar({ + options: { map: toolMap, defaultValue: defaultTool }, + toolGroupId: [toolGroupId, volumeToolGroupId], +}); + +addSegmentIndexDropdown(segmentationId); + +addDropdownToToolbar({ + options: { + values: viewportIds, + }, + onSelectedValueChange: (value) => { + activeViewport = renderingEngine.getViewport(value); + ai.initViewport(activeViewport); + }, +}); + +addButtonToToolbar({ + title: 'Cache', + onClick: () => { + if (cached !== undefined) { + return; + } + cached = false; + ai.cacheImageEncodings(); + }, +}); + +/** + * Runs the demo + */ +async function run() { + // Get the load started here, as it can take a while. + ai.initModel(); + + // Init Cornerstone and related libraries + await initDemo(); + + // Define tool groups to add the segmentation display tool to + const toolGroup = ToolGroupManager.createToolGroup(toolGroupId); + addManipulationBindings(toolGroup, { toolMap }); + // The threshold circle has preview turned on by default, so use it as the + // tool to get/apply previews with. + toolForPreview = toolGroup.getToolInstance('ThresholdCircle'); + + const volumeToolGroup = ToolGroupManager.createToolGroup(volumeToolGroupId); + addManipulationBindings(volumeToolGroup, { toolMap }); + + // Get Cornerstone imageIds and fetch metadata into RAM + const imageIdsFull = await createImageIdsAndCacheMetaData({ + StudyInstanceUID: + '1.3.6.1.4.1.14519.5.2.1.7009.2403.334240657131972136850343327463', + SeriesInstanceUID: + '1.3.6.1.4.1.14519.5.2.1.7009.2403.226151125820845824875394858561', + wadoRsRoot: + getLocalUrl() || 'https://d3t6nz73ql33tx.cloudfront.net/dicomweb', + }); + + const imageIds = imageIdsFull.reverse(); // .slice(35, 45); + // Instantiate a rendering engine + const renderingEngineId = 'myRenderingEngine'; + renderingEngine = new RenderingEngine(renderingEngineId); + + // Create the viewports + const viewportInputArray = [ + { + viewportId: viewportId, + type: ViewportType.STACK, + element: element0, + defaultOptions: { + background: [0.2, 0, 0.2], + }, + }, + { + viewportId: viewportIds[1], + type: ViewportType.ORTHOGRAPHIC, + element: element1, + defaultOptions: { + orientation: Enums.OrientationAxis.AXIAL, + background: [0.2, 0.2, 0], + }, + }, + { + viewportId: viewportIds[2], + type: ViewportType.ORTHOGRAPHIC, + element: element2, + defaultOptions: { + orientation: Enums.OrientationAxis.SAGITTAL, + background: [0, 0.2, 0], + }, + }, + { + viewportId: viewportIds[3], + type: ViewportType.ORTHOGRAPHIC, + element: element3, + defaultOptions: { + orientation: Enums.OrientationAxis.CORONAL, + background: [0.2, 0, 0.2], + }, + }, + ]; + + renderingEngine.setViewports(viewportInputArray); + toolGroup.addViewport(viewportId, renderingEngineId); + volumeToolGroup.addViewport(viewportIds[1], renderingEngineId); + volumeToolGroup.addViewport(viewportIds[2], renderingEngineId); + volumeToolGroup.addViewport(viewportIds[3], renderingEngineId); + + // Get the stack viewport that was created + viewport = renderingEngine.getViewport(viewportId); + activeViewport = viewport; + + // Add a segmentation that will contains the contour annotations + const segmentationImages = + await imageLoader.createAndCacheDerivedLabelmapImages(imageIds); + + const segmentationImageIds = segmentationImages.map((image) => image.imageId); + + // Set the stack on the viewport + await viewport.setStack(imageIds); + viewport.setOptions(ONNXSegmentationController.viewportOptions); + + // This init model is waiting for completion, whereas the earlier one just + // starts loading in the background. + await ai.initModel(); + // Connect the default viewport here to start things off - requires the initModel to be done + ai.initViewport(viewport, toolForPreview); + + const volume = await volumeLoader.createAndCacheVolume(volumeId, { + imageIds, + }); + + volume.load(); + + await setVolumesForViewports( + renderingEngine, + [{ volumeId }], + viewportIds.slice(1) + ); + + // Render the image + + segmentation.addSegmentations([ + { + segmentationId, + representation: { + type: csToolsEnums.SegmentationRepresentations.Labelmap, + data: { + imageIds: segmentationImageIds, + }, + }, + }, + ]); + + const segMap = { + [viewportIds[0]]: [{ segmentationId }], + [viewportIds[1]]: [{ segmentationId }], + [viewportIds[2]]: [{ segmentationId }], + [viewportIds[3]]: [{ segmentationId }], + }; + + // Create a segmentation representation associated to the toolGroupId + await segmentation.addLabelmapRepresentationToViewportMap(segMap); + + elements.forEach((element) => + element.addEventListener(csToolsEnums.Events.KEY_DOWN, (evt) => { + const { key } = evt.detail; + const { element } = activeViewport; + if (key === 'Escape') { + cornerstoneTools.cancelActiveManipulations(element); + toolForPreview.rejectPreview(element); + } else if (key === 'Enter') { + toolForPreview.acceptPreview(element); + } else if (key === 'n') { + ai.interpolateScroll(activeViewport, 1); + } + }) + ); + + // volumeViewport.setViewReference(viewport.getViewReference()); + // volumeViewport.setViewPresentation(viewport.getViewPresentation()); +} + +run(); diff --git a/packages/ai/package.json b/packages/ai/package.json new file mode 100644 index 0000000000..23bcc4cbee --- /dev/null +++ b/packages/ai/package.json @@ -0,0 +1,59 @@ +{ + "name": "@cornerstonejs/ai", + "version": "2.5.1", + "description": "AI and ML Interfaces for Cornerstone3D", + "files": [ + "dist" + ], + "module": "./dist/esm/index.js", + "types": "./dist/esm/index.d.ts", + "directories": { + "build": "dist" + }, + "exports": { + ".": { + "import": "./dist/esm/index.js", + "types": "./dist/esm/index.d.ts" + } + }, + "publishConfig": { + "access": "public" + }, + "scripts": { + "test": "jest --testTimeout 60000", + "clean": "rimraf dist", + "build": "yarn run build:esm", + "build:esm": "tsc --project ./tsconfig.json", + "build:esm:watch": "tsc --project ./tsconfig.json --watch", + "dev": "tsc --project ./tsconfig.json --watch", + "build:all": "yarn run build:esm", + "build:update-api": "yarn run build:esm && api-extractor run --local", + "start": "tsc --project ./tsconfig.json --watch", + "format": "prettier --write 'src/**/*.js' 'test/**/*.js'", + "lint": "eslint --fix ." + }, + "repository": { + "type": "git", + "url": "git+https://github.com/dcmjs-org/dcmjs.git" + }, + "author": "@cornerstonejs", + "license": "MIT", + "bugs": { + "url": "https://github.com/cornerstonejs/cornerstone3D/issues" + }, + "homepage": "https://github.com/cornerstonejs/cornerstone3D/blob/main/packages/ai/README.md", + "dependencies": { + "@babel/runtime-corejs2": "^7.17.8", + "buffer": "^6.0.3", + "dcmjs": "^0.29.8", + "gl-matrix": "^3.4.3", + "lodash.clonedeep": "^4.5.0", + "ndarray": "^1.0.19", + "onnxruntime-common": "1.17.1", + "onnxruntime-web": "1.17.1" + }, + "peerDependencies": { + "@cornerstonejs/core": "^2.5.1", + "@cornerstonejs/tools": "^2.5.1" + } +} diff --git a/packages/ai/src/ONNXSegmentationController.ts b/packages/ai/src/ONNXSegmentationController.ts new file mode 100644 index 0000000000..f2c915ee34 --- /dev/null +++ b/packages/ai/src/ONNXSegmentationController.ts @@ -0,0 +1,1221 @@ +import type { Types } from '@cornerstonejs/core'; +import { utilities, eventTarget, Enums } from '@cornerstonejs/core'; +import * as cornerstoneTools from '@cornerstonejs/tools'; +import type { Types as cstTypes } from '@cornerstonejs/tools'; + +// @ts-ignore +import ort from 'onnxruntime-web/webgpu'; +import { vec3 } from 'gl-matrix'; + +const { annotation } = cornerstoneTools; +const { state: annotationState } = annotation; +const { Events } = Enums; +const { Events: toolsEvents } = cornerstoneTools.Enums; + +const { segmentation } = cornerstoneTools; +const { filterAnnotationsForDisplay } = cornerstoneTools.utilities.planar; + +const { triggerSegmentationDataModified } = + segmentation.triggerSegmentationEvents; + +/** + * clone tensor + */ +function cloneTensor(t) { + return new ort.Tensor(t.type, Float32Array.from(t.data), t.dims); +} + +/* + * create feed for the original facebook model + */ +function feedForSam(emb, points, labels, modelSize = [1024, 1024]) { + const maskInput = new ort.Tensor( + new Float32Array(256 * 256), + [1, 1, 256, 256] + ); + const hasMask = new ort.Tensor(new Float32Array([0]), [1]); + const originalImageSize = new ort.Tensor(new Float32Array(modelSize), [2]); + const pointCoords = new ort.Tensor(new Float32Array(points), [ + 1, + points.length / 2, + 2, + ]); + const pointLabels = new ort.Tensor(new Float32Array(labels), [ + 1, + labels.length, + ]); + + const key = (emb.image_embeddings && 'image_embeddings') || 'embeddings'; + return { + image_embeddings: cloneTensor(emb[key]), + point_coords: pointCoords, + point_labels: pointLabels, + mask_input: maskInput, + has_mask_input: hasMask, + orig_im_size: originalImageSize, + }; +} + +/* + Create a function which will be passed to the promise + and resolve it when FileReader has finished loading the file. + */ +function getBuffer(fileData) { + return new Promise((resolve) => { + const reader = new FileReader(); + reader.readAsArrayBuffer(fileData); + reader.onload = function () { + const arrayBuffer = reader.result as ArrayBuffer; + const bytes = new Float32Array(arrayBuffer); + resolve(bytes); + }; + }); +} + +export enum Loggers { + Log = 'status', + Encoder = 'encoder', + Decoder = 'decoder', +} + +/** + * The ONNXController handles the interaction between CS3D viewports and ONNX segmentation + * models to allow segmentation of volume and stack viewports using browser local + * data models. The process is that a particular view of the viewport is rendered + * using the loadImageToCanvas to generate the required model size. This will render + * without annotations/segmentation. Then, this view is passed to the encoder model which + * transforms the data into a set of information about the overall image. This encoding + * can take a while, so it is cached. + * + * To generate segmentations, the encoded model data is combined with information from the + * user in the form of annotations on the image to include or exclude regions from the segmentation, + * and allow the segmentation to be guided. + * + * Once the segmentation data has been generated, it is converted from the overlay/bitmap model into + * a CS3D segmentation map, in the segment index currently being worked on. + * + * The encoded model data is stored in browser local storage, and each model + * typically consumes about 4 mb per frame. The path for the storage is + * based on the target id and the study/series/instance attributes. That + * path is: + * + * `///` + * where the file path is the instance UID, or a made up name based on the + * slice index and view normal for orthographic images. + * + * For encoding images, there are two sessions to consider. The currently + * displaying session has information about the image being worked on, while a + * second session allows encoding images not being currently viewed. This allows + * for background encoding of images. However, note the library does NOT allow + * encoding two images at the same time, it is merely that two images can be + * queued up for encoding at the same time and can have non-overlapping results. + */ +export default class ONNXSegmentationController { + /** Default name for a tool for inclusion points */ + public static MarkerInclude = 'MarkerInclude'; + /** Default name for a tool for exclusion points */ + public static MarkerExclude = 'MarkerExclude'; + + /** Some viewport options for loadImageToCanvas */ + public static viewportOptions = { + displayArea: { + storeAsInitialCamera: true, + // Use nearest neighbour for better results on the model + interpolationType: Enums.InterpolationType.NEAREST, + imageArea: [1, 1], + imageCanvasPoint: { + // TODO - fix this so top left corner works + imagePoint: [0.5, 0.5], + canvasPoint: [0.5, 0.5], + }, + } as Types.DisplayArea, + background: [0, 0, 0.2], + }; + // the image size on canvas + maxWidth = 1024; + maxHeight = 1024; + + // the image size supported by the model + modelWidth = 1024; + modelHeight = 1024; + + /** + * Defines the URL endpoints and render sizes/setup for the various models that + * can be used. + */ + static MODELS = { + sam_l: [ + { + name: 'sam-l-encoder', + url: '/sam_l/vit_l_encoder.onnx', + size: 1224, + key: 'encoder', + feedType: 'images', + }, + { + name: 'sam-l-decoder', + url: '/sam_l/vit_l_decoder.onnx', + size: 17, + key: 'decoder', + }, + ], + sam_h: [ + { + name: 'sam-h-encoder', + url: '/sam_h/vit_h_encoder.onnx', + size: 18, + key: 'encoder', + feedType: 'images', + }, + { + name: 'sam-h-decoder', + url: '/sam_h/vit_h_decoder.onnx', + size: 1, + key: 'decoder', + }, + ], + }; + + public canvas = document.createElement('canvas'); + public canvasMask = document.createElement('canvas'); + + /** Store other sessions to be used for next images. */ + private sessions = []; + private config; + private points = []; + private labels = []; + + private loadingAI: Promise; + + protected viewport; + protected excludeTool = ONNXSegmentationController.MarkerExclude; + protected tool; + protected currentImage; + private listeners = [console.log]; + protected desiredImage = { + imageId: null, + sampleImageId: null, + imageIndex: -1, + decoder: null, + encoder: null, + }; + + protected imageEncodings = new Map(); + protected sharedImageEncoding; + protected boxRadius = 5; + protected imageImageData; + protected isGpuInUse = false; + protected annotationsNeedUpdating = false; + protected maskImageData; + protected promptAnnotationTypes = [ + ONNXSegmentationController.MarkerInclude, + ONNXSegmentationController.MarkerExclude, + ]; + /** The type name of the preview tool used for the accept/reject labelmap preview */ + protected previewToolType = 'ThresholdCircle'; + + /** + * Configure the ML Controller. No parameters are required, and will default + * to the basic set of controls using MarkerInclude/Exclude and the default SAM + * model for segmentation. + * + * The plan is to add additional prompt types and default models which can be + * applied, but this version is a simple/basic version. + * + * @param options - a set of options to configure this with + * * listeners - a set of functions to call to get log type feedback on the status of the segmentation + * * getPromptAnnotations - a function to get the annotations which are used as input to the AI segmentation + * * promptAnnotationTypes - a list of annotation type names to use to generate the prompts. This is an + * alternate to the getPromptAnnotations + * * models - a set of model configurations to run - note these are added globally to the static models + * * modelName - the specific model name to choose. Must exist in models. + */ + constructor( + options = { + listeners: null, + getPromptAnnotations: null, + promptAnnotationTypes: null, + models: null, + modelName: null, + previewToolType: 'ThresholdCircle', + } + ) { + if (options.listeners) { + this.listeners = [...options.listeners]; + } + if (options.getPromptAnnotations) { + this.getPromptAnnotations = options.getPromptAnnotations; + } + this.promptAnnotationTypes = + options.promptAnnotationTypes || this.promptAnnotationTypes; + if (options.models) { + Object.assign(ONNXSegmentationController.MODELS, options.models); + } + this.previewToolType = options.previewToolType || this.previewToolType; + this.config = this.getConfig(options.modelName); + } + + /** + * Loads the AI model. This can take a while and will return a promise + * which resolves when the model is completed. If the model is already loaded, + * then the promise returned will be resolved already. Can safely be called multiple + * times to allow starting the model load early, and then waiting for it when done. + */ + public initModel(): Promise { + if (!this.loadingAI) { + this.loadingAI = this.load(); + } + return this.loadingAI; + } + + /** + * Connects a viewport up to get anotations and updates + * Note that only one viewport at a time is permitted as the model needs to + * load data about the active viewport. This method will disconnect a previous + * viewport automatically. + * + * The viewport must have a labelmap segmentation registered, as well as a + * tool which extendds LabelmapBaseTool to use for setting the preview view + * once the decode is completed. This is provided as toolForPreview + * + * @param viewport - a viewport to listen for annotations and rendered events + * @param toolForPreview - this tool is used to access the preview object and + * create a new preview instance. + */ + public initViewport(viewport) { + const { desiredImage } = this; + if (this.viewport) { + this.disconnectViewport(this.viewport); + } + this.currentImage = null; + this.viewport = viewport; + const toolGroup = cornerstoneTools.ToolGroupManager.getToolGroupForViewport( + viewport.id, + viewport.getRenderingEngine()?.id + ); + this.tool = toolGroup.getToolInstance(this.previewToolType); + + desiredImage.imageId = + viewport.getCurrentImageId() || viewport.getReferenceId(); + if (desiredImage.imageId.startsWith('volumeId:')) { + desiredImage.sampleImageId = viewport.getImageIds( + viewport.getVolumeId() + )[0]; + } else { + desiredImage.sampleImageId = desiredImage.imageId; + } + + viewport.element.addEventListener( + Events.IMAGE_RENDERED, + this.viewportRenderedListener + ); + const boundListener = this.annotationModifiedListener; + eventTarget.addEventListener(toolsEvents.ANNOTATION_ADDED, boundListener); + eventTarget.addEventListener( + toolsEvents.ANNOTATION_MODIFIED, + boundListener + ); + eventTarget.addEventListener( + toolsEvents.ANNOTATION_COMPLETED, + boundListener + ); + if (desiredImage.imageId) { + this.tryLoad(); + } + } + + /** + * The interpolateScroll checks to see if there are any annotations on the + * current image in the specified viewport, and if so, scrolls in the given + * direction and copies the annotations to the new image displayed. + * It will not copy any annotations onto a viewport already containing + * prompt annotations, nor will it do anything if there are no annotations + * on the current viewport. + * + * Assumes the current and next viewports have the same viewport normal, and + * that the difference between positions is entirely contained by the difference + * in the focal point between viewports. This difference is added to each + * point in the annotations. + */ + public async interpolateScroll(viewport = this.viewport, dir = 1) { + const { element } = viewport; + this.tool.acceptPreview(element); + const promptAnnotations = this.getPromptAnnotations(viewport); + + if (!promptAnnotations.length) { + return; + } + + const currentSliceIndex = viewport.getCurrentImageIdIndex(); + const { focalPoint } = viewport.getCamera(); + const viewRef = viewport.getViewReference({ + sliceIndex: currentSliceIndex + dir, + }); + if (!viewRef || viewRef.sliceIndex === currentSliceIndex) { + console.warn('No next image in direction', dir, currentSliceIndex); + return; + } + + viewport.scroll(dir); + // Wait for the scroll to complete + await new Promise((resolve) => window.setTimeout(resolve, 250)); + const nextAnnotations = this.getPromptAnnotations(viewport); + if (nextAnnotations.length > 0) { + return; + } + + // Add the difference between the new and old focal point as being the + // position difference between images. Does not account for any + // rotational differences between frames that may occur on stacks + const { focalPoint: newFocal } = viewport.getCamera(); + const newDelta = vec3.sub(vec3.create(), newFocal as vec3, focalPoint); + for (const annotation of promptAnnotations) { + annotation.interpolationUID ||= crypto.randomUUID(); + const newAnnotation = structuredClone(annotation); + newAnnotation.annotationUID = undefined; + Object.assign(newAnnotation.metadata, viewRef); + // @ts-ignore + newAnnotation.cachedStats = {}; + for (const handle of newAnnotation.data.handles.points) { + vec3.add(handle, handle, newDelta); + } + annotationState.addAnnotation(newAnnotation, viewport.element); + } + viewport.render(); + } + + /** + * Logs the message to the given log level + */ + protected log(logger: Loggers, ...args) { + for (const listener of this.listeners) { + listener(logger, ...args); + } + } + + /** + * Gets a list of the include/exclude orientation annotations applying to the + * current image id. + */ + protected getPromptAnnotations = (viewport = this.viewport) => { + const annotations = []; + const { element } = viewport; + for (const annotationName of this.promptAnnotationTypes) { + annotations.push( + ...annotationState.getAnnotations(annotationName, element) + ); + } + const currentAnnotations = filterAnnotationsForDisplay( + this.viewport, + annotations + ); + return currentAnnotations; + }; + + /** + * A listener for viewport being rendered that tried loading/encoding the + * new image if it is different from the previous image. Will return before + * the image is encoded. Can be called without binding as it is already + * bound to the this object. + * The behaviour of the callback is that if the image has changed in terms + * of which image (new view reference), then that image is set as the + * currently desired encoded image, and a new encoding will be read from + * cache or one will be created and stored in cache. + * + * This does not need to be manually bound, the initViewport will bind + * this to the correct rendering messages. + */ + protected viewportRenderedListener = (_event) => { + const { viewport, currentImage, desiredImage } = this; + desiredImage.imageId = + viewport.getCurrentImageId() || viewport.getReferenceId(); + desiredImage.imageIndex = viewport.getCurrentImageIdIndex(); + if (!desiredImage.imageId) { + return; + } + if (desiredImage.imageId.startsWith('volumeId:')) { + desiredImage.sampleImageId = viewport.getImageIds( + viewport.getVolumeId() + )[0]; + } else { + desiredImage.sampleImageId = desiredImage.imageId; + } + if (desiredImage.imageId === currentImage?.imageId) { + return; + } + const { canvasMask } = this; + const ctxMask = canvasMask.getContext('2d'); + ctxMask.clearRect(0, 0, canvasMask.width, canvasMask.height); + + this.tryLoad({ resetImage: true }); + }; + + /** + * This is an already bound annotation modified listener, that is added/removed + * from the viewport by the initViewport method. + * This listener does the following: + * * Gets the annotations, returning immediately if there are no annotations + * * Marks the annotations as needing an update + * * Starts an encoding on the current image if it is not already encoded + * * When the image is encoded, runs the decoder + * * Once the decoder has completed, converts the results into a CS3D segmentation preview + * + * Note that the decoder run will not occur if the image is changed before the + * decoder starts running, and that encoding a new image may not start until + * an ongoing decoder operations has completed. + */ + protected annotationModifiedListener = (_event?) => { + const currentAnnotations = this.getPromptAnnotations(); + if (!currentAnnotations.length) { + return; + } + this.annotationsNeedUpdating = true; + this.tryLoad(); + }; + + /** + * Disconnects the given viewport, removing the listeners. + */ + public disconnectViewport(viewport) { + viewport.element.removeEventListener( + Events.IMAGE_RENDERED, + this.viewportRenderedListener + ); + const boundListener = this.annotationModifiedListener; + eventTarget.removeEventListener( + toolsEvents.ANNOTATION_MODIFIED, + boundListener + ); + eventTarget.removeEventListener( + toolsEvents.ANNOTATION_COMPLETED, + boundListener + ); + } + + /** + * Does the actual load, separated from the public method to allow starting + * the AI to load and then waiting for it once other things are also ready. + * This is done internally so that only a single load/setup is created, allowing + * for the load to be started and only waited for when other things are ready. + */ + protected async load() { + const { sessions } = this; + this.canvas.style.cursor = 'wait'; + + let loader; + // Create two sessions, one for the current images, and a second session + // for caching non-visible images. This doesn't create two GPU sessions, + // but does create two sessions for storage of encoded results. + for (let i = 0; i < 2; i++) { + sessions.push({ + sessionIndex: i, + encoder: null, + decoder: null, + imageEmbeddings: null, + isLoading: false, + canvas: i === 0 ? this.canvas : document.createElement('canvas'), + }); + if (i === 0) { + loader = this.loadModels( + ONNXSegmentationController.MODELS[this.config.model], + sessions[i] + ).catch((e) => { + this.log(Loggers.Log, "Couldn't load models", e); + }); + await loader; + } else { + // Only the encoder is needed otherwise + sessions[i].encoder = sessions[0].encoder; + } + sessions[i].loader = loader; + } + } + + /** + * Clears the points, labels and annotations related to the ML model from the + * viewport. + */ + public clear(viewport) { + this.points = []; + this.labels = []; + this.getPromptAnnotations(viewport).forEach((annotation) => + annotationState.removeAnnotation(annotation.annotationUID) + ); + } + + /** + * Cache the next image encoded. This will start at the current image id, + * and will keep on fetching additional images, wrapping round to the 0...current + * position-1 so as to fetch all images. + * Works with both volume (orthographic) and stack viewports. + * This will interfere with any image navigation + * + * @param current - the starting image near which other images should be cached. + * @param offset - what offset to the current image should be used, that is, + * for 125 images, if the current was 5, and the offset is 6, then the + * image at sliceIndex 5+6+1 will be used. + * @param length - the number of images. This will be determined dynamically + * based on when the view reference for the next slice returns undefined. + * Defaults to 1,000,000 to start with. + */ + public async cacheImageEncodings( + current = this.viewport.getCurrentImageIdIndex(), + offset = 0, + length = 1000_000 + ) { + const { viewport, imageEncodings } = this; + if (offset >= length) { + // We are done. + return; + } + const index = (offset + current) % length; + const view = viewport.getViewReference({ sliceIndex: index }); + if (!view) { + length = index; + return this.cacheImageEncodings(current, offset, length); + } + const imageId = + view.referencedImageId || viewport.getReferenceId({ sliceIndex: index }); + if (!imageEncodings.has(imageId)) { + // Try loading from storage + await this.loadStorageImageEncoding(current, imageId, index); + } + if (imageEncodings.has(imageId)) { + this.cacheImageEncodings(current, offset + 1, length); + return; + } + // Try doing a load, so that UI has priority + this.tryLoad(); + if (this.isGpuInUse) { + setTimeout(() => this.cacheImageEncodings(current, offset), 500); + return; + } + + this.log(Loggers.Log, 'Caching', index, imageId); + const sampleImageId = viewport.getImageIds()[0]; + this.handleImage({ imageId, sampleImageId }, this.sessions[1]).then(() => { + this.cacheImageEncodings(current, offset + 1, length); + }); + } + + /** + * Handles a new image. This will render the image to a separate canvas + * using the load image to canvas, and then will load or generate encoder + * values into the imageSession provided. + * If there is already an image being handled or worked on, returns immediately. + * + * At the end of the handle, tries calling the tryLoad method to see if there + * are other high priority tasks to complete. + */ + protected async handleImage({ imageId, sampleImageId }, imageSession) { + if (imageId === imageSession.imageId || this.isGpuInUse) { + return; + } + const { viewport, desiredImage } = this; + this.isGpuInUse = true; + imageSession.imageId = imageId; + imageSession.sampleImageId = sampleImageId; + try { + const isCurrent = desiredImage.imageId === imageId; + const { canvas } = imageSession; + if (isCurrent) { + this.log( + Loggers.Encoder, + `Loading image on ${imageSession.sessionIndex}` + ); + this.log(Loggers.Decoder, 'Awaiting image'); + canvas.style.cursor = 'wait'; + } + this.points = []; + this.labels = []; + const width = this.maxWidth; + const height = this.maxHeight; + canvas.width = width; + canvas.height = height; + imageSession.imageEmbeddings = undefined; + const size = canvas.style.width; + + const ctx = canvas.getContext('2d', { willReadFrequently: true }); + ctx.clearRect(0, 0, width, height); + + const renderArguments = { + canvas, + imageId, + viewportOptions: { + ...viewport.defaultOptions, + ...ONNXSegmentationController.viewportOptions, + }, + viewReference: null, + renderingEngineId: viewport.getRenderingEngine().id, + }; + if (imageId.startsWith('volumeId:')) { + const viewRef = viewport.getViewReference(); + renderArguments.viewReference = viewRef; + renderArguments.imageId = null; + } + imageSession.canvasPosition = await utilities.loadImageToCanvas( + renderArguments + ); + canvas.style.width = size; + canvas.style.height = size; + if (isCurrent) { + this.log( + Loggers.Encoder, + `Rendered image on ${imageSession.sessionIndex}` + ); + } + + this.imageImageData = ctx.getImageData(0, 0, width, height); + + const data = await this.restoreImageEncoding(imageSession, imageId); + if (data) { + imageSession.imageEmbeddings = data; + if (desiredImage.imageId === imageId) { + this.log(Loggers.Encoder, 'Cached Image'); + canvas.style.cursor = 'default'; + } + } else { + const t = await ort.Tensor.fromImage(this.imageImageData, { + resizedWidth: this.modelWidth, + resizedHeight: this.modelHeight, + }); + const { feedType = 'input_image' } = this.config.encoder; + const feed = (feedType === 'images' && { images: t }) || + (feedType === 'pixelValues' && { pixel_values: t }) || { + input_image: t, + }; + await imageSession.loader; + const session = await imageSession.encoder; + if (!session) { + this.log(Loggers.Log, '****** No session'); + return; + } + const start = performance.now(); + imageSession.imageEmbeddings = session.run(feed); + const data = await imageSession.imageEmbeddings; + this.storeImageEncoding(imageSession, imageId, data); + if (desiredImage.imageId === imageId) { + this.log( + Loggers.Encoder, + `Image Ready ${imageSession.sessionIndex} ${( + performance.now() - start + ).toFixed(1)} ms` + ); + canvas.style.cursor = 'default'; + } + } + } finally { + this.isGpuInUse = false; + } + + this.tryLoad(); + } + + /** + * This method tries to run the decode that wraps the decode operation in + * checks for whether hte GPU is in use, whether the decode has otherwise completed, + * and a general try/catch around the decode. + */ + protected async runDecode() { + const { canvas } = this; + if (this.isGpuInUse || !this.currentImage?.imageEmbeddings) { + return; + } + this.isGpuInUse = true; + try { + this.canvas.style.cursor = 'wait'; + await this.decode(this.points, this.labels); + } finally { + canvas.style.cursor = 'default'; + this.isGpuInUse = false; + } + } + + /** + * This function will try setting the current image to the desired loading image + * if it isn't already the desired one, and invoke the handleImage. + * If the desired image is already the right one, then it will try to run + * and outstanding decoder task. + * This sequence allows out of order decodes to happen and to start the latest + * encode/decode at the time the last operation has completed. If the user + * performs multiple operations, then only the last set is handled. + */ + public tryLoad(options = { resetImage: false }) { + const { viewport, desiredImage } = this; + if (!desiredImage.imageId || options.resetImage) { + desiredImage.imageId = + viewport.getCurrentImageId() || viewport.getReferenceId(); + this.currentImage = null; + } + // Always use session 0 for the current session + const [session] = this.sessions; + + if (session.imageId === desiredImage.imageId) { + if (this.currentImage !== session) { + this.currentImage = session; + } + this.updateAnnotations(); + return; + } + this.handleImage(desiredImage, session); + } + + /** + * Maps world points to destination points. + * Assumes the destination canvas is scale to fit at 100% in both dimensions, + * while the source point is also assumed to be in the same position, but not + * the same scale. + */ + mapAnnotationPoint(worldPoint) { + const { viewport } = this; + const canvasPoint = viewport.worldToCanvas(worldPoint); + const { width, height } = viewport.canvas; + const { width: destWidth, height: destHeight } = this.canvas; + + const x = Math.trunc( + (canvasPoint[0] * destWidth * devicePixelRatio) / width + ); + const y = Math.trunc( + (canvasPoint[1] * destHeight * devicePixelRatio) / height + ); + return [x, y]; + } + + /** + * Updates annotations when they have changed in some way, running the decoder. + * This will mark the annotations as needing update, so that if the + * encoding of the image isn't ready yet, or the encoder is otherwise busy, + * it will run the update again once the tryLoad is done at the end of the task. + */ + updateAnnotations() { + if ( + this.isGpuInUse || + !this.annotationsNeedUpdating || + !this.currentImage + ) { + return; + } + const currentAnnotations = this.getPromptAnnotations(); + this.annotationsNeedUpdating = false; + this.points = []; + this.labels = []; + if (!currentAnnotations?.length) { + return; + } + for (const annotation of currentAnnotations) { + const handle = annotation.data.handles.points[0]; + const point = this.mapAnnotationPoint(handle); + const label = annotation.metadata.toolName === this.excludeTool ? 0 : 1; + this.points.push(point[0]); + this.points.push(point[1]); + this.labels.push(label); + } + this.runDecode(); + } + + /** + * Restores a stored image encoding from memory cache first, and from + * the browser storage secondly. This is much faster than re-generating it + * all the time. + */ + async restoreImageEncoding(session, imageId) { + if (!this.sharedImageEncoding) { + return; + } + if (!this.imageEncodings.has(imageId)) { + await this.loadStorageImageEncoding(session, imageId); + } + const floatData = this.imageEncodings.get(imageId); + if (floatData) { + const key = + (this.sharedImageEncoding.image_embeddings && 'image_embeddings') || + 'embeddings'; + this.sharedImageEncoding[key].cpuData.set(floatData); + return this.sharedImageEncoding; + } + } + + /** + * Loads the image encoding from browser storage. + */ + async loadStorageImageEncoding(session, imageId, index = null) { + try { + const root = await this.getDirectoryForImageId(session, imageId); + const name = this.getFileNameForImageId(imageId, this.config.model); + if (!root || !name) { + return null; + } + const fileHandle = await findFileEntry(root, name); + if (!fileHandle) { + return null; + } + this.log(Loggers.Log, 'Loading from storage', index || imageId, name); + const file = await fileHandle.getFile(); + if (file) { + const buffer = await getBuffer(file); + this.imageEncodings.set(imageId, buffer); + } + } catch (e) { + this.log(Loggers.Log, 'Unable to fetch file', imageId, e); + } + } + + /** + * Stores the image encoding to both memory cache and browser storage. + */ + async storeImageEncoding(session, imageId, data) { + if (!this.sharedImageEncoding) { + this.sharedImageEncoding = data; + } + const storeData = (data.image_embeddings || data.embeddings)?.cpuData; + if (!storeData) { + console.log('Unable to store data', data); + return; + } + const writeData = new Float32Array(storeData); + this.imageEncodings.set(imageId, writeData); + try { + const root = await this.getDirectoryForImageId(session, imageId); + const name = this.getFileNameForImageId(imageId, this.config.model); + if (!root || !name) { + return; + } + const fileHandle = await root.getFileHandle(name, { create: true }); + const writable = await fileHandle.createWritable(); + await writable.write(writeData); + await writable.close(); + // Note, this data is not considered for persistence as it is assumed multiple + // series are being worked on. See the persistence model for adding ahead of + // time caching. + } catch (e) { + this.log(Loggers.Log, 'Unable to write', imageId, e); + } + } + + /** + * Given the mask created by the AI model, assigns the data to a new preview + * instance of a labelmap and triggers the modified event so that the new + * segmentation data is visible. Replaces existing segmentation on that + * image. + */ + createLabelmap(mask, canvasPosition, _points, _labels) { + const { canvas, viewport } = this; + const preview = this.tool.addPreview(viewport.element); + const { previewSegmentIndex, memo, segmentationId } = preview; + const previewVoxelManager = + memo?.voxelManager || preview.previewVoxelManager; + const { dimensions } = previewVoxelManager; + const { data } = mask; + + const { origin, topRight, bottomLeft } = canvasPosition; + const downVec = vec3.subtract(vec3.create(), bottomLeft, origin); + const rightVec = vec3.subtract(vec3.create(), topRight, origin); + // Vectors are scaled to unit vectors in CANVAS space + vec3.scale(downVec, downVec, 1 / canvas.height); + vec3.scale(rightVec, rightVec, 1 / canvas.width); + + const worldPointJ = vec3.create(); + const worldPoint = vec3.create(); + const imageData = viewport.getDefaultImageData(); + + // Assumes that the load to canvas size is bigger than the destination + // size - if that isnt true, then this should super-sample the data + for (let j = 0; j < canvas.height; j++) { + vec3.scaleAndAdd(worldPointJ, origin, downVec, j); + for (let i = 0; i < canvas.width; i++) { + vec3.scaleAndAdd(worldPoint, worldPointJ, rightVec, i); + const ijkPoint = imageData.worldToIndex(worldPoint).map(Math.round); + if ( + ijkPoint.findIndex((v, index) => v < 0 || v >= dimensions[index]) !== + -1 + ) { + continue; + } + // 4 values - RGBA - per pixel + const maskIndex = 4 * (i + j * this.maxWidth); + const v = data[maskIndex]; + if (v > 0) { + previewVoxelManager.setAtIJKPoint(ijkPoint, previewSegmentIndex); + } else { + previewVoxelManager.setAtIJKPoint(ijkPoint, null); + } + } + } + triggerSegmentationDataModified(segmentationId); + } + + /** + * Runs the GPU decoder operation itself. + */ + async decode(points, labels, useSession = this.currentImage) { + const { canvas, canvasMask, imageImageData, desiredImage, boxRadius } = + this; + const ctx = canvas.getContext('2d', { willReadFrequently: true }); + ctx.clearRect(0, 0, canvas.width, canvas.height); + canvas.width = imageImageData.width; + canvas.height = imageImageData.height; + canvasMask.width = imageImageData.width; + canvasMask.height = imageImageData.height; + + if (!useSession || useSession.imageId !== desiredImage.imageId) { + this.log( + Loggers.Log, + '***** Image not current, need to wait for current image' + ); + return; + } + + // Comment this line out to draw just the overlay mask data + ctx.putImageData(imageImageData, 0, 0); + + if (points.length) { + // need to wait for encoder to be ready + if (!useSession.imageEmbeddings) { + await useSession.encoder; + } + + // wait for encoder to deliver embeddings + const emb = await useSession.imageEmbeddings; + + // the decoder + const session = useSession.decoder; + + const feed = feedForSam(emb, points, labels); + const start = performance.now(); + const res = await session.run(feed); + this.log( + Loggers.Decoder, + `decoder ${useSession.sessionIndex} ${( + performance.now() - start + ).toFixed(1)} ms` + ); + + for (let i = 0; i < points.length; i += 2) { + const label = labels[i / 2]; + ctx.fillStyle = label ? 'blue' : 'pink'; + + ctx.fillRect( + points[i] - boxRadius, + points[i + 1] - boxRadius, + 2 * boxRadius, + 2 * boxRadius + ); + } + const mask = res.masks; + this.maskImageData = mask.toImageData(); + this.createLabelmap( + this.maskImageData, + useSession.canvasPosition, + points, + labels + ); + ctx.globalAlpha = 0.3; + const { data } = this.maskImageData; + const counts = []; + for (let i = 0; i < data.length; i += 4) { + const v = data[i]; + if (v > 0) { + if (v < 255) { + data[i] = 0; + if (v > 192) { + data[i + 1] = 255; + } else { + data[i + 2] = v + 64; + } + } + counts[v] = 1 + (counts[v] || 0); + } + } + const bitmap = await createImageBitmap(this.maskImageData); + ctx.drawImage(bitmap, 0, 0); + + const ctxMask = canvasMask.getContext('2d'); + ctxMask.globalAlpha = 0.9; + ctxMask.drawImage(bitmap, 0, 0); + } + } + + /* + * fetch and cache the ONNX model at the given url/name. + */ + async fetchAndCacheModel(url, name) { + try { + const cache = await caches.open('onnx'); + let cachedResponse = await cache.match(url); + if (cachedResponse == undefined) { + await cache.add(url); + cachedResponse = await cache.match(url); + this.log(Loggers.Log, `${name} (network)`); + } else { + this.log(Loggers.Log, `${name} (cached)`); + } + const data = await cachedResponse.arrayBuffer(); + return data; + } catch (error) { + this.log(Loggers.Log, `${name} (network)`); + return await fetch(url).then((response) => response.arrayBuffer()); + } + } + + /* + * load model cache data and creates an instance. This calls fetchAndCacheModle + * once for the decoder and encoder, and then instantiates an instance. + */ + async loadModels(models, imageSession = this.currentImage) { + const cache = await caches.open('onnx'); + let missing = 0; + const urls = []; + // Get the list of urls to download + // eslint-disable-next-line @typescript-eslint/no-explicit-any + for (const model of Object.values(models) as any[]) { + const cachedResponse = await cache.match(model.url); + if (cachedResponse === undefined) { + missing += model.size; + } + urls.push(model.url); + } + if (missing > 0) { + this.log( + Loggers.Log, + `downloading ${missing} MB from network ... it might take a while` + ); + } else { + this.log(Loggers.Log, 'loading...'); + } + const start = performance.now(); + for (const model of Object.values(models) as any[]) { + const opt = { + executionProviders: [this.config.provider], + enableMemPattern: false, + enableCpuMemArena: false, + extra: { + session: { + disable_prepacking: '1', + use_device_allocator_for_initializers: '1', + use_ort_model_bytes_directly: '1', + use_ort_model_bytes_for_initializers: '1', + }, + }, + interOpNumThreads: 4, + intraOpNumThreads: 2, + }; + const model_bytes = await this.fetchAndCacheModel(model.url, model.name); + const extra_opt = model.opt || {}; + const sessionOptions = { ...opt, ...extra_opt }; + this.config[model.key] = model; + imageSession[model.key] = await ort.InferenceSession.create( + model_bytes, // `http://localhost:4000${model.url}`, + sessionOptions + ); + } + const stop = performance.now(); + this.log( + Loggers.Log, + `ready, ${(stop - start).toFixed(1)}ms`, + urls.join(', ') + ); + } + + /** + * Gets the storage directory for storing the given image id + */ + async getDirectoryForImageId(session, imageId) { + if ( + imageId.indexOf('/studies/') === -1 || + imageId.indexOf('/instances/') === -1 + ) { + imageId = session.sampleImageId; + if ( + !imageId || + imageId.indexOf('/studies/') === -1 || + imageId.indexOf('/instances/') === -1 + ) { + return null; + } + } + const studySeriesUids = imageId + .split('/studies/')[1] + .split('/instances/')[0] + .split('/'); + const [studyUID, _series, seriesUID] = studySeriesUids; + const root = await window.navigator.storage.getDirectory(); + const modelRoot = await getOrCreateDir(root, this.config.model); + const studyRoot = await getOrCreateDir(modelRoot, studyUID); + const seriesRoot = await getOrCreateDir(studyRoot, seriesUID); + return seriesRoot; + } + + /** + * Gets the storage file name for the given imageId + */ + getFileNameForImageId(imageId, extension) { + if (imageId.startsWith('volumeId:')) { + const sliceIndex = imageId.indexOf('sliceIndex='); + const focalPoint = imageId.indexOf('&focalPoint='); + const name = imageId + .substring(sliceIndex, focalPoint) + .replace('&', '.') + .replace('sliceIndex=', 'volume.'); + return name + extension; + } + const instancesLocation = imageId.indexOf('/instances/'); + if (instancesLocation != -1) { + const sopLocation = instancesLocation + 11; + const nextSlash = imageId.indexOf('/', sopLocation); + return imageId.substring(sopLocation, nextSlash) + extension; + } + } + + /** + * Creates a configuration for which encoder/decoder to run. TODO - move + * this into the constructor. + */ + getConfig(modelName = 'sam_b') { + if (this.config) { + return this.config; + } + const query = window.location.search.substring(1); + const config = { + model: modelName, + provider: 'webgpu', + device: 'gpu', + threads: 4, + local: null, + isSlimSam: false, + }; + const vars = query.split('&'); + for (let i = 0; i < vars.length; i++) { + const pair = vars[i].split('='); + if (pair[0] in config) { + config[pair[0]] = decodeURIComponent(pair[1]); + } else if (pair[0].length > 0) { + throw new Error('unknown argument: ' + pair[0]); + } + } + config.threads = parseInt(String(config.threads)); + config.local = parseInt(config.local); + ort.env.wasm.wasmPaths = 'dist/'; + ort.env.wasm.numThreads = config.threads; + ort.env.wasm.proxy = config.provider == 'wasm'; + + this.config = config; + return config; + } +} + +/** Gets or creates a storage directory */ +async function getOrCreateDir(dir, name) { + return ( + (await findFileEntry(dir, name)) || + dir.getDirectoryHandle(name, { create: true }) + ); +} + +/** Finds a file entry in the given directory. */ +async function findFileEntry(dir, name) { + for await (const [key, value] of dir) { + if (key === name) { + return value; + } + } +} diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts new file mode 100644 index 0000000000..2eb5b10ff7 --- /dev/null +++ b/packages/ai/src/index.ts @@ -0,0 +1,3 @@ +import ONNXSegmentationController from './ONNXSegmentationController'; + +export { ONNXSegmentationController }; diff --git a/packages/ai/tsconfig.json b/packages/ai/tsconfig.json new file mode 100644 index 0000000000..bd62237b47 --- /dev/null +++ b/packages/ai/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist/esm", + "rootDir": "./src" + }, + "include": ["./src/**/*"], +} diff --git a/packages/core/src/RenderingEngine/VolumeViewport.ts b/packages/core/src/RenderingEngine/VolumeViewport.ts index 31ad863527..03ba3dbf0f 100644 --- a/packages/core/src/RenderingEngine/VolumeViewport.ts +++ b/packages/core/src/RenderingEngine/VolumeViewport.ts @@ -32,6 +32,7 @@ import getImageSliceDataForVolumeViewport from '../utilities/getImageSliceDataFo import { transformCanvasToIJK } from '../utilities/transformCanvasToIJK'; import { transformIJKToCanvas } from '../utilities/transformIJKToCanvas'; import type vtkMapper from '@kitware/vtk.js/Rendering/Core/Mapper'; +import getVolumeViewportScrollInfo from '../utilities/getVolumeViewportScrollInfo'; /** * An object representing a VolumeViewport. VolumeViewports are used to render @@ -442,37 +443,29 @@ class VolumeViewport extends BaseVolumeViewport { } /** - * Returns the imageId index of the current slice in the volume viewport. - * Note: this is not guaranteed to be the same as the slice index in the view - * To get the slice index in the view (scroll position), use `getSliceIndex()` + * Uses the slice range information to compute the current image id index. + * Note that this may be offset from the origin location, or opposite in + * direction to the distance from the origin location, as the index is a + * complete index from minimum to maximum. * - * In future we will even delete this method as it should not be used - * at all. + * @returns The slice index in the direction of the view. This index is in + * the same position/size/direction as the scroll utility. That is, + * ```scroll(dir)``` + * and + * ```viewport.setView(viewport.getView({sliceIndex: viewport.getCurrentImageIdIndex()+dir}))``` * - * @returns The slice index in the direction of the view + * have the same affect, excluding end/looping conditions. */ - public getCurrentImageIdIndex = (volumeId?: string): number => { - const { viewPlaneNormal, focalPoint } = this.getCamera(); - - const imageData = this.getImageData(volumeId); - - if (!imageData) { - return; - } - - const { origin, direction, spacing } = imageData; - - const spacingInNormal = getSpacingInNormalDirection( - { direction, spacing }, - viewPlaneNormal + public getCurrentImageIdIndex = ( + volumeId?: string, + useSlabThickness = true + ): number => { + const { currentStepIndex } = getVolumeViewportScrollInfo( + this, + volumeId || this.getVolumeId(), + useSlabThickness ); - const sub = vec3.create(); - vec3.sub(sub, focalPoint, origin); - const distance = vec3.dot(sub, viewPlaneNormal); - - // divide by the spacing in the normal direction to get the - // number of steps, and subtract 1 to get the index - return Math.round(Math.abs(distance) / spacingInNormal); + return currentStepIndex; }; /** diff --git a/packages/core/src/RenderingEngine/helpers/addImageSlicesToViewports.ts b/packages/core/src/RenderingEngine/helpers/addImageSlicesToViewports.ts index 092eaec466..f992b0f43e 100644 --- a/packages/core/src/RenderingEngine/helpers/addImageSlicesToViewports.ts +++ b/packages/core/src/RenderingEngine/helpers/addImageSlicesToViewports.ts @@ -16,7 +16,7 @@ import type { * @param immediateRender - If true, the volumes will be rendered immediately * @returns A promise that resolves when all volumes have been added */ -async function addImageSlicesToViewports( +function addImageSlicesToViewports( renderingEngine: IRenderingEngine, stackInputs: IStackInput[], viewportIds: string[] @@ -39,13 +39,10 @@ async function addImageSlicesToViewports( } } - const addStackPromises = viewportIds.map(async (viewportId) => { + viewportIds.forEach((viewportId) => { const viewport = renderingEngine.getStackViewport(viewportId); - viewport.addImages(stackInputs); }); - - await Promise.all(addStackPromises); } export default addImageSlicesToViewports; diff --git a/packages/core/src/utilities/VoxelManager.ts b/packages/core/src/utilities/VoxelManager.ts index 034710e195..df9a09f98f 100644 --- a/packages/core/src/utilities/VoxelManager.ts +++ b/packages/core/src/utilities/VoxelManager.ts @@ -746,11 +746,11 @@ export default class VoxelManager { } const _getConstructor = () => { - const pixelInfo = getPixelInfo(0); - if (!pixelInfo?.pixelData) { + const { voxelManager: imageVoxelManager } = getPixelInfo(0); + if (!imageVoxelManager) { return null; } - return pixelInfo.pixelData.constructor; + return imageVoxelManager.getConstructor(); }; const voxelManager = new VoxelManager(dimensions, { @@ -849,18 +849,18 @@ export default class VoxelManager { let maxValue = -Infinity; for (let sliceIndex = 0; sliceIndex < dimensions[2]; sliceIndex++) { - const { pixelData } = getPixelInfo( + const { voxelManager: imageVoxelManager } = getPixelInfo( (sliceIndex * sliceSize) / numberOfComponents ); - if (pixelData && SliceDataConstructor) { + if (imageVoxelManager && SliceDataConstructor) { const sliceStart = sliceIndex * sliceSize; const sliceEnd = sliceStart + sliceSize; // @ts-ignore const sliceData = new SliceDataConstructor(sliceSize); // @ts-ignore sliceData.set(scalarData.subarray(sliceStart, sliceEnd)); - pixelData.set(sliceData); + imageVoxelManager.scalarData = sliceData; // Update min/max values for this slice for (let i = 0; i < sliceData.length; i++) { @@ -1098,6 +1098,8 @@ export default class VoxelManager { scalarData[index] = v; return isChanged; }, + _getConstructor: () => + scalarData.constructor as new (length: number) => PixelDataTypedArray, _id: '_createNumberVolumeVoxelManager', }); voxels.scalarData = scalarData; diff --git a/packages/tools/src/enums/StrategyCallbacks.ts b/packages/tools/src/enums/StrategyCallbacks.ts index c0bb140c60..c182dad5c7 100644 --- a/packages/tools/src/enums/StrategyCallbacks.ts +++ b/packages/tools/src/enums/StrategyCallbacks.ts @@ -48,6 +48,13 @@ enum StrategyCallbacks { // Internal Details INTERNAL_setValue = 'setValue', + /** + * Adds a preview interpolation from the given data. This allows external + * methods to set/update the preview and then have it shown/accepted in the + * normal fashion. + */ + AddPreview = 'addPreview', + /** inner circle size */ ComputeInnerCircleRadius = 'computeInnerCircleRadius', diff --git a/packages/tools/src/tools/annotation/DragProbeTool.ts b/packages/tools/src/tools/annotation/DragProbeTool.ts index 33f1481c2d..8df7c4c915 100644 --- a/packages/tools/src/tools/annotation/DragProbeTool.ts +++ b/packages/tools/src/tools/annotation/DragProbeTool.ts @@ -15,6 +15,8 @@ import type { SVGDrawingHelper, ToolProps, } from '../../types'; +import { ChangeTypes, Events } from '../../enums'; + import triggerAnnotationRenderForViewportIds from '../../utilities/triggerAnnotationRenderForViewportIds'; import ProbeTool from './ProbeTool'; import type { ProbeAnnotation } from '../../types/ToolSpecificAnnotationTypes'; @@ -159,7 +161,7 @@ class DragProbeTool extends ProbeTool { if ( !data.cachedStats[targetId] || - data.cachedStats[targetId].value == null + (data.cachedStats[targetId] as Record).value === null ) { data.cachedStats[targetId] = { Modality: null, diff --git a/packages/tools/src/tools/annotation/LivewireContourTool.ts b/packages/tools/src/tools/annotation/LivewireContourTool.ts index 58399d0bc3..91e3b42cee 100644 --- a/packages/tools/src/tools/annotation/LivewireContourTool.ts +++ b/packages/tools/src/tools/annotation/LivewireContourTool.ts @@ -964,7 +964,7 @@ class LivewireContourTool extends ContourSegmentationBaseTool { if ( !data.cachedStats[targetId] || - data.cachedStats[targetId].areaUnit == null + (data.cachedStats[targetId] as Record)?.areaUnit === null ) { data.cachedStats[targetId] = { Modality: null, diff --git a/packages/tools/src/tools/annotation/PlanarFreehandROITool.ts b/packages/tools/src/tools/annotation/PlanarFreehandROITool.ts index f7960cf29c..146dde6fc6 100644 --- a/packages/tools/src/tools/annotation/PlanarFreehandROITool.ts +++ b/packages/tools/src/tools/annotation/PlanarFreehandROITool.ts @@ -33,7 +33,10 @@ import type { } from '../../types'; import { triggerAnnotationModified } from '../../stateManagement/annotation/helpers/state'; import { drawLinkedTextBox } from '../../drawingSvg'; -import type { PlanarFreehandROIAnnotation } from '../../types/ToolSpecificAnnotationTypes'; +import type { + ContourAnnotation, + PlanarFreehandROIAnnotation, +} from '../../types/ToolSpecificAnnotationTypes'; import { getTextBoxCoordsCanvas } from '../../utilities/drawing'; import type { PlanarFreehandROICommonData } from '../../utilities/math/polyline/planarFreehandROIInternalTypes'; @@ -534,7 +537,9 @@ class PlanarFreehandROITool extends ContourSegmentationBaseTool { return false; } - protected createAnnotation(evt: EventTypes.InteractionEventType): Annotation { + protected createAnnotation( + evt: EventTypes.InteractionEventType + ): ContourAnnotation { const worldPos = evt.detail.currentPoints.world; const contourAnnotation = super.createAnnotation(evt); diff --git a/packages/tools/src/tools/annotation/ProbeTool.ts b/packages/tools/src/tools/annotation/ProbeTool.ts index 7927ecc203..7f3de45ff2 100644 --- a/packages/tools/src/tools/annotation/ProbeTool.ts +++ b/packages/tools/src/tools/annotation/ProbeTool.ts @@ -25,7 +25,7 @@ import { drawTextBox as drawTextBoxSvg, } from '../../drawingSvg'; import { state } from '../../store/state'; -import { Events } from '../../enums'; +import { ChangeTypes, Events } from '../../enums'; import { getViewportIdsWithToolToRender } from '../../utilities/viewportFilters'; import { resetElementCursor, @@ -107,18 +107,23 @@ class ProbeTool extends AnnotationTool { isDrawing: boolean; isHandleOutsideImage: boolean; - constructor( - toolProps: PublicToolProps = {}, - defaultToolProps: ToolProps = { - supportedInteractionTypes: ['Mouse', 'Touch'], - configuration: { - shadow: true, - preventHandleOutsideImage: false, - getTextLines: defaultGetTextLines, - }, - } - ) { - super(toolProps, defaultToolProps); + public static probeDefaults = { + supportedInteractionTypes: ['Mouse', 'Touch'], + configuration: { + shadow: true, + preventHandleOutsideImage: false, + getTextLines: defaultGetTextLines, + }, + }; + + constructor(toolProps: PublicToolProps = {}, defaultToolProps?) { + super( + toolProps, + AnnotationTool.mergeDefaultProps( + ProbeTool.probeDefaults, + defaultToolProps + ) + ); } // Not necessary for this tool but needs to be defined since it's an abstract @@ -473,7 +478,7 @@ class ProbeTool extends AnnotationTool { if ( !data.cachedStats[targetId] || - data.cachedStats[targetId].value == null + data.cachedStats[targetId].value === null ) { data.cachedStats[targetId] = { Modality: null, @@ -481,7 +486,12 @@ class ProbeTool extends AnnotationTool { value: null, }; - this._calculateCachedStats(annotation, renderingEngine, enabledElement); + this._calculateCachedStats( + annotation, + renderingEngine, + enabledElement, + ChangeTypes.StatsUpdated + ); } else if (annotation.invalidated) { this._calculateCachedStats(annotation, renderingEngine, enabledElement); @@ -566,7 +576,12 @@ class ProbeTool extends AnnotationTool { return renderStatus; }; - _calculateCachedStats(annotation, renderingEngine, enabledElement) { + _calculateCachedStats( + annotation, + renderingEngine, + enabledElement, + changeType = ChangeTypes.StatsUpdated + ) { const data = annotation.data; const { renderingEngineId, viewport } = enabledElement; const { element } = viewport; @@ -663,7 +678,7 @@ class ProbeTool extends AnnotationTool { annotation.invalidated = false; // Dispatching annotation modified - triggerAnnotationModified(annotation, element); + triggerAnnotationModified(annotation, element, changeType); } return cachedStats; @@ -674,7 +689,7 @@ function defaultGetTextLines(data, targetId): string[] { const cachedVolumeStats = data.cachedStats[targetId]; const { index, value, modalityUnit } = cachedVolumeStats; - if (value === undefined) { + if (value === undefined || !index) { return; } diff --git a/packages/tools/src/tools/annotation/SplineROITool.ts b/packages/tools/src/tools/annotation/SplineROITool.ts index fadcaebcee..99578f07c5 100644 --- a/packages/tools/src/tools/annotation/SplineROITool.ts +++ b/packages/tools/src/tools/annotation/SplineROITool.ts @@ -43,7 +43,10 @@ import { getCalibratedLengthUnitsAndScale } from '../../utilities/getCalibratedU import getMouseModifierKey from '../../eventDispatchers/shared/getMouseModifier'; import { ContourWindingDirection } from '../../types/ContourAnnotation'; -import type { SplineROIAnnotation } from '../../types/ToolSpecificAnnotationTypes'; +import type { + ContourAnnotation, + SplineROIAnnotation, +} from '../../types/ToolSpecificAnnotationTypes'; import type { AnnotationModifiedEventDetail, ContourAnnotationCompletedEventDetail, @@ -851,7 +854,9 @@ class SplineROITool extends ContourSegmentationBaseTool { points.push(polyline[polyline.length - 1]); } - protected createAnnotation(evt: EventTypes.InteractionEventType): Annotation { + protected createAnnotation( + evt: EventTypes.InteractionEventType + ): ContourAnnotation { const contourAnnotation = super.createAnnotation(evt); const { world: worldPos } = evt.detail.currentPoints; const { type: splineType } = this.configuration.spline; diff --git a/packages/tools/src/tools/base/AnnotationDisplayTool.ts b/packages/tools/src/tools/base/AnnotationDisplayTool.ts index dcbaa5a9b3..0ead78e96b 100644 --- a/packages/tools/src/tools/base/AnnotationDisplayTool.ts +++ b/packages/tools/src/tools/base/AnnotationDisplayTool.ts @@ -1,16 +1,19 @@ import { utilities, getEnabledElement, - StackViewport, cache, - VideoViewport, BaseVolumeViewport, } from '@cornerstonejs/core'; import type { Types } from '@cornerstonejs/core'; import BaseTool from './BaseTool'; import { getAnnotationManager } from '../../stateManagement/annotation/annotationState'; -import type { Annotation, Annotations, SVGDrawingHelper } from '../../types'; +import type { + Annotation, + Annotations, + EventTypes, + SVGDrawingHelper, +} from '../../types'; import triggerAnnotationRender from '../../utilities/triggerAnnotationRender'; import filterAnnotationsForDisplay from '../../utilities/planar/filterAnnotationsForDisplay'; import { getStyleProperty } from '../../stateManagement/annotation/config/helpers'; @@ -123,6 +126,59 @@ abstract class AnnotationDisplayTool extends BaseTool { }); }; + /** + * Creates an annotation containing the basic data set. + */ + protected createAnnotation(evt: EventTypes.InteractionEventType): Annotation { + const eventDetail = evt.detail; + const { currentPoints, element } = eventDetail; + const { world: worldPos } = currentPoints; + + const enabledElement = getEnabledElement(element); + const { viewport } = enabledElement; + + const camera = viewport.getCamera(); + const { viewPlaneNormal, viewUp, position: cameraPosition } = camera; + + const referencedImageId = this.getReferencedImageId( + viewport, + worldPos, + viewPlaneNormal, + viewUp + ); + + const viewReference = viewport.getViewReference({ points: [worldPos] }); + + return { + highlighted: true, + invalidated: true, + metadata: { + toolName: this.getToolName(), + ...viewReference, + referencedImageId, + viewUp, + cameraPosition, + }, + data: { + cachedStats: {}, + handles: { + points: [], + activeHandleIndex: null, + textBox: { + hasMoved: false, + worldPosition: [0, 0, 0], + worldBoundingBox: { + topLeft: [0, 0, 0], + topRight: [0, 0, 0], + bottomLeft: [0, 0, 0], + bottomRight: [0, 0, 0], + }, + }, + }, + }, + }; + } + protected getReferencedImageId( viewport: Types.IViewport, worldPos: Types.Point3, diff --git a/packages/tools/src/tools/base/BaseTool.ts b/packages/tools/src/tools/base/BaseTool.ts index dc6e20b118..3c7820ee21 100644 --- a/packages/tools/src/tools/base/BaseTool.ts +++ b/packages/tools/src/tools/base/BaseTool.ts @@ -1,4 +1,4 @@ -import { utilities, BaseVolumeViewport } from '@cornerstonejs/core'; +import { utilities } from '@cornerstonejs/core'; import type { Types } from '@cornerstonejs/core'; import ToolModes from '../../enums/ToolModes'; import type StrategyCallbacks from '../../enums/StrategyCallbacks'; @@ -27,8 +27,25 @@ abstract class BaseTool { */ protected memo: utilities.HistoryMemo.Memo; + /** + * Has the defaults associated with the base tool. + */ + static defaults = { + configuration: { + strategies: {}, + defaultStrategy: undefined, + activeStrategy: undefined, + strategyOptions: {}, + }, + }; + constructor(toolProps: PublicToolProps, defaultToolProps: ToolProps) { - const initialProps = utilities.deepMerge(defaultToolProps, toolProps); + const mergedDefaults = BaseTool.mergeDefaultProps( + BaseTool.defaults, + defaultToolProps + ); + + const initialProps = utilities.deepMerge(mergedDefaults, toolProps); const { configuration = {}, @@ -36,20 +53,36 @@ abstract class BaseTool { toolGroupId, } = initialProps; - // If strategies are not initialized in the tool config - if (!configuration.strategies) { - configuration.strategies = {}; - configuration.defaultStrategy = undefined; - configuration.activeStrategy = undefined; - configuration.strategyOptions = {}; - } - this.toolGroupId = toolGroupId; this.supportedInteractionTypes = supportedInteractionTypes || []; this.configuration = Object.assign({}, configuration); this.mode = ToolModes.Disabled; } + /** + * Does a deep merge of property options. Allows extending the default values + * for a child class. + * + * @param defaultProps - this is a base set of defaults to merge into + * @param additionalProps - the additional properties to merge into the default props + * + * @returns defaultProps if additional props not defined, or a merge into a new object + * containing additionalProps adding onto and overriding defaultProps. + */ + public static mergeDefaultProps(defaultProps = {}, additionalProps?) { + if (!additionalProps) { + return defaultProps; + } + return utilities.deepMerge(defaultProps, additionalProps); + } + + /** + * Newer method for getting the tool name as a property + */ + public get toolName() { + return this.getToolName(); + } + /** * Returns the name of the tool * @returns The name of the tool. diff --git a/packages/tools/src/tools/base/ContourBaseTool.ts b/packages/tools/src/tools/base/ContourBaseTool.ts index 8c4391ed30..8865be226c 100644 --- a/packages/tools/src/tools/base/ContourBaseTool.ts +++ b/packages/tools/src/tools/base/ContourBaseTool.ts @@ -101,61 +101,26 @@ abstract class ContourBaseTool extends AnnotationTool { return renderStatus; } - protected createAnnotation(evt: EventTypes.InteractionEventType): Annotation { - const eventDetail = evt.detail; - const { currentPoints, element } = eventDetail; - const { world: worldPos } = currentPoints; - - const enabledElement = getEnabledElement(element); - const { viewport } = enabledElement; - - const camera = viewport.getCamera(); - const { viewPlaneNormal, viewUp, position: cameraPosition } = camera; - - const referencedImageId = this.getReferencedImageId( - viewport, - worldPos, - viewPlaneNormal, - viewUp - ); - - const viewReference = viewport.getViewReference({ points: [worldPos] }); - - return { - highlighted: true, - invalidated: true, - metadata: { - toolName: this.getToolName(), - ...viewReference, - referencedImageId, - viewUp, - cameraPosition, - }, - data: { - handles: { - points: [], - activeHandleIndex: null, - textBox: { - hasMoved: false, - worldPosition: [0, 0, 0], - worldBoundingBox: { - topLeft: [0, 0, 0], - topRight: [0, 0, 0], - bottomLeft: [0, 0, 0], - bottomRight: [0, 0, 0], - }, - }, - }, - contour: { - polyline: [], - closed: false, - }, + /** + * Extends the base annotation with contour annotation information, specifically + * the contour object in the data, and the interpolation information. + */ + protected createAnnotation( + evt: EventTypes.InteractionEventType + ): ContourAnnotation { + const annotation = super.createAnnotation(evt); + Object.assign(annotation.data, { + contour: { + polyline: [], + closed: false, }, + }); + Object.assign(annotation, { interpolationUID: '', autoGenerated: false, - }; + }); + return annotation as ContourAnnotation; } - /** * Add the annotation to the annotation manager. * @param annotation - Contour annotation that is being added diff --git a/packages/tools/src/tools/base/ContourSegmentationBaseTool.ts b/packages/tools/src/tools/base/ContourSegmentationBaseTool.ts index bec1bbfe6f..05aaea7fab 100644 --- a/packages/tools/src/tools/base/ContourSegmentationBaseTool.ts +++ b/packages/tools/src/tools/base/ContourSegmentationBaseTool.ts @@ -7,14 +7,17 @@ import type { AnnotationRenderContext, } from '../../types'; -import { getSegmentation } from '../../stateManagement/segmentation/getSegmentation'; import type { ContourSegmentationAnnotation } from '../../types/ContourSegmentationAnnotation'; -import type { SplineContourSegmentationAnnotation } from '../../types/ToolSpecificAnnotationTypes'; +import type { + SplineContourSegmentationAnnotation, + ContourAnnotation, +} from '../../types/ToolSpecificAnnotationTypes'; import type { StyleSpecifier } from '../../types/AnnotationStyle'; import { SegmentationRepresentations } from '../../enums'; import ContourBaseTool from './ContourBaseTool'; import { triggerSegmentationDataModified } from '../../stateManagement/segmentation/triggerSegmentationEvents'; import InterpolationManager from '../../utilities/segmentation/InterpolationManager/InterpolationManager'; + import { addContourSegmentationAnnotation, removeContourSegmentationAnnotation, @@ -58,7 +61,12 @@ abstract class ContourSegmentationBaseTool extends ContourBaseTool { return true; } - protected createAnnotation(evt: EventTypes.InteractionEventType): Annotation { + /** + * Creates a contour segmentation annotation + */ + protected createAnnotation( + evt: EventTypes.InteractionEventType + ): ContourAnnotation { const eventDetail = evt.detail; const { element } = eventDetail; diff --git a/packages/tools/src/tools/displayTools/Labelmap/addLabelmapToElement.ts b/packages/tools/src/tools/displayTools/Labelmap/addLabelmapToElement.ts index 3d1dd41165..adc7a465eb 100644 --- a/packages/tools/src/tools/displayTools/Labelmap/addLabelmapToElement.ts +++ b/packages/tools/src/tools/displayTools/Labelmap/addLabelmapToElement.ts @@ -98,7 +98,7 @@ async function addLabelmapToElement( ]; // Add labelmap volumes to the viewports to be be rendered, but not force the render - await addImageSlicesToViewports(renderingEngine, stackInputs, [viewportId]); + addImageSlicesToViewports(renderingEngine, stackInputs, [viewportId]); } // Just to make sure if the segmentation data had value before, it gets updated too diff --git a/packages/tools/src/tools/segmentation/BrushTool.ts b/packages/tools/src/tools/segmentation/BrushTool.ts index 2aed4f0000..ad14d09c51 100644 --- a/packages/tools/src/tools/segmentation/BrushTool.ts +++ b/packages/tools/src/tools/segmentation/BrushTool.ts @@ -1,23 +1,13 @@ -import { - utilities as csUtils, - cache, - getEnabledElement, - StackViewport, - eventTarget, - Enums, - BaseVolumeViewport, - volumeLoader, -} from '@cornerstonejs/core'; +import { getEnabledElement } from '@cornerstonejs/core'; +import type { Types } from '@cornerstonejs/core'; import { vec3, vec2 } from 'gl-matrix'; -import type { Types } from '@cornerstonejs/core'; import type { PublicToolProps, ToolProps, EventTypes, SVGDrawingHelper, } from '../../types'; -import { BaseTool } from '../base'; import { fillInsideSphere, thresholdInsideSphere, @@ -29,12 +19,7 @@ import { fillInsideCircle, } from './strategies/fillCircle'; import { eraseInsideCircle } from './strategies/eraseCircle'; -import { - Events, - ToolModes, - SegmentationRepresentations, - StrategyCallbacks, -} from '../../enums'; +import { Events, ToolModes, StrategyCallbacks } from '../../enums'; import { drawCircle as drawCircleSvg } from '../../drawingSvg'; import { resetElementCursor, @@ -42,70 +27,15 @@ import { } from '../../cursors/elementCursor'; import triggerAnnotationRenderForViewportUIDs from '../../utilities/triggerAnnotationRenderForViewportIds'; -import type { LabelmapSegmentationDataVolume } from '../../types/LabelmapTypes'; -import { - getCurrentLabelmapImageIdForViewport, - getSegmentation, - getStackSegmentationImageIdsForViewport, -} from '../../stateManagement/segmentation/segmentationState'; -import { getLockedSegmentIndices } from '../../stateManagement/segmentation/segmentLocking'; -import { getActiveSegmentIndex } from '../../stateManagement/segmentation/getActiveSegmentIndex'; -import { getSegmentIndexColor } from '../../stateManagement/segmentation/config/segmentationColor'; -import type vtkImageData from '@kitware/vtk.js/Common/DataModel/ImageData'; -import { getActiveSegmentation } from '../../stateManagement/segmentation/getActiveSegmentation'; - -/** - * A type for preview data/information, used to setup previews on hover, or - * maintain the preview information. - */ -export type PreviewData = { - /** - * The preview data returned from the strategy - */ - preview: unknown; - timer?: number; - timerStart: number; - startPoint: Types.Point2; - element: HTMLDivElement; - isDrag: boolean; -}; +import LabelmapBaseTool from './LabelmapBaseTool'; /** * @public */ -class BrushTool extends BaseTool { +class BrushTool extends LabelmapBaseTool { static toolName; - private _editData: { - override: { - voxelManager: Types.IVoxelManager; - imageData: vtkImageData; - }; - segmentsLocked: number[]; // - imageId?: string; // stack labelmap - imageIds?: string[]; // stack labelmap - volumeId?: string; // volume labelmap - referencedVolumeId?: string; - } | null; - private _hoverData?: { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - brushCursor: any; - segmentationId: string; - segmentIndex: number; - segmentColor: [number, number, number, number]; - viewportIdsToRender: string[]; - centerCanvas?: Array; - viewport: Types.IViewport; - }; - - private _previewData?: PreviewData = { - preview: null, - element: null, - timerStart: 0, - timer: null, - startPoint: [NaN, NaN], - isDrag: false, - }; + prg; constructor( toolProps: PublicToolProps = {}, defaultToolProps: ToolProps = { @@ -198,141 +128,6 @@ class BrushTool extends BaseTool { this.rejectPreview(); } - createEditData(element) { - const enabledElement = getEnabledElement(element); - const { viewport } = enabledElement; - - const activeSegmentation = getActiveSegmentation(viewport.id); - if (!activeSegmentation) { - const event = new CustomEvent(Enums.Events.ERROR_EVENT, { - detail: { - type: 'Segmentation', - message: - 'No active segmentation detected, create a segmentation representation before using the brush tool', - }, - cancelable: true, - }); - eventTarget.dispatchEvent(event); - return null; - } - - const { segmentationId } = activeSegmentation; - - const segmentsLocked = getLockedSegmentIndices(segmentationId); - - const { representationData } = getSegmentation(segmentationId); - - if (viewport instanceof BaseVolumeViewport) { - const { volumeId } = representationData[ - SegmentationRepresentations.Labelmap - ] as LabelmapSegmentationDataVolume; - const actors = viewport.getActors(); - - const isStackViewport = viewport instanceof StackViewport; - - if (isStackViewport) { - const event = new CustomEvent(Enums.Events.ERROR_EVENT, { - detail: { - type: 'Segmentation', - message: 'Cannot perform brush operation on the selected viewport', - }, - cancelable: true, - }); - eventTarget.dispatchEvent(event); - return null; - } - - // we used to take the first actor here but we should take the one that is - // probably the same size as the segmentation volume - const volumes = actors.map((actorEntry) => - cache.getVolume(actorEntry.referencedId) - ); - - const segmentationVolume = cache.getVolume(volumeId); - - const referencedVolumeIdToThreshold = - volumes.find((volume) => - csUtils.isEqual(volume.dimensions, segmentationVolume.dimensions) - )?.volumeId || volumes[0]?.volumeId; - - return { - volumeId, - referencedVolumeId: - this.configuration.thresholdVolumeId ?? referencedVolumeIdToThreshold, - segmentsLocked, - }; - } else { - const segmentationImageId = getCurrentLabelmapImageIdForViewport( - viewport.id, - segmentationId - ); - - if (!segmentationImageId) { - // if there is no stack segmentation slice for the current image - // we should not allow the user to perform any operation - return; - } - - // I hate this, but what can you do sometimes - if (this.configuration.activeStrategy.includes('SPHERE')) { - const referencedImageIds = viewport.getImageIds(); - const isValidVolumeForSphere = - csUtils.isValidVolume(referencedImageIds); - - if (!isValidVolumeForSphere) { - throw new Error( - 'Volume is not reconstructable for sphere manipulation' - ); - } - - const volumeId = `${segmentationId}_${viewport.id}`; - const volume = cache.getVolume(volumeId); - if (volume) { - return { - imageId: segmentationImageId, - segmentsLocked, - override: { - voxelManager: volume.voxelManager, - imageData: volume.imageData, - }, - }; - } else { - const labelmapImageIds = getStackSegmentationImageIdsForViewport( - viewport.id, - segmentationId - ); - - if (!labelmapImageIds || labelmapImageIds.length === 1) { - return { - imageId: segmentationImageId, - segmentsLocked, - }; - } - - // it will return the cached volume if it already exists - const volume = volumeLoader.createAndCacheVolumeFromImagesSync( - volumeId, - labelmapImageIds - ); - - return { - imageId: segmentationImageId, - segmentsLocked, - override: { - voxelManager: volume.voxelManager, - imageData: volume.imageData, - }, - }; - } - } else { - return { - imageId: segmentationImageId, - segmentsLocked, - }; - } - } - } - preMouseDownCallback = ( evt: EventTypes.MouseDownActivateEventType ): boolean => { @@ -437,70 +232,6 @@ class BrushTool extends BaseTool { ); }; - private createHoverData(element, centerCanvas?) { - const enabledElement = getEnabledElement(element); - const { viewport } = enabledElement; - - const camera = viewport.getCamera(); - const { viewPlaneNormal, viewUp } = camera; - - const viewportIdsToRender = [viewport.id]; - - const { segmentIndex, segmentationId, segmentColor } = - this.getActiveSegmentationData(viewport) || {}; - - // Center of circle in canvas Coordinates - const brushCursor = { - metadata: { - viewPlaneNormal: [...viewPlaneNormal], - viewUp: [...viewUp], - FrameOfReferenceUID: viewport.getFrameOfReferenceUID(), - referencedImageId: '', - toolName: this.getToolName(), - segmentColor, - }, - data: {}, - }; - - return { - brushCursor, - centerCanvas, - segmentIndex, - viewport, - segmentationId, - segmentColor, - viewportIdsToRender, - }; - } - - private getActiveSegmentationData(viewport) { - const viewportId = viewport.id; - const activeRepresentation = getActiveSegmentation(viewportId); - - if (!activeRepresentation) { - return; - } - - const { segmentationId } = activeRepresentation; - const segmentIndex = getActiveSegmentIndex(segmentationId); - - if (!segmentIndex) { - return; - } - - const segmentColor = getSegmentIndexColor( - viewportId, - segmentationId, - segmentIndex - ); - - return { - segmentIndex, - segmentationId, - segmentColor, - }; - } - /** * Updates the cursor position and whether it is showing or not. * Can be over-ridden to add more cursor details or a preview. @@ -560,31 +291,6 @@ class BrushTool extends BaseTool { this._previewData.startPoint = currentPoints.canvas; }; - protected getOperationData(element?) { - const editData = this._editData || this.createEditData(element); - const { segmentIndex, segmentationId, brushCursor } = - this._hoverData || this.createHoverData(element); - const { data, metadata = {} } = brushCursor || {}; - const { viewPlaneNormal, viewUp } = metadata; - const operationData = { - ...editData, - points: data?.handles?.points, - segmentIndex, - previewColors: this.configuration.preview.enabled - ? this.configuration.preview.previewColors - : null, - viewPlaneNormal, - toolGroupId: this.toolGroupId, - segmentationId, - viewUp, - strategySpecificConfiguration: - this.configuration.strategySpecificConfiguration, - // Provide the preview information so that data can be used directly - preview: this._previewData?.preview, - }; - return operationData; - } - private _calculateCursor(element, centerCanvas) { const enabledElement = getEnabledElement(element); const { viewport } = enabledElement; @@ -710,41 +416,6 @@ class BrushTool extends BaseTool { return stats; } - /** - * Cancels any preview view being shown, resetting any segments being shown. - */ - public rejectPreview(element = this._previewData.element) { - if (!element || !this._previewData.preview) { - return; - } - const enabledElement = getEnabledElement(element); - this.applyActiveStrategyCallback( - enabledElement, - this.getOperationData(element), - StrategyCallbacks.RejectPreview - ); - this._previewData.preview = null; - this._previewData.isDrag = false; - } - - /** - * Accepts a preview, marking it as the active segment. - */ - public acceptPreview(element = this._previewData.element) { - if (!element) { - return; - } - const enabledElement = getEnabledElement(element); - - this.applyActiveStrategyCallback( - enabledElement, - this.getOperationData(element), - StrategyCallbacks.AcceptPreview - ); - this._previewData.isDrag = false; - this._previewData.preview = null; - } - /** * Add event handlers for the modify event loop, and prevent default event propagation. */ diff --git a/packages/tools/src/tools/segmentation/LabelmapBaseTool.ts b/packages/tools/src/tools/segmentation/LabelmapBaseTool.ts new file mode 100644 index 0000000000..066f4b3436 --- /dev/null +++ b/packages/tools/src/tools/segmentation/LabelmapBaseTool.ts @@ -0,0 +1,390 @@ +import { + getEnabledElement, + cache, + utilities as csUtils, + Enums, + eventTarget, + BaseVolumeViewport, + volumeLoader, +} from '@cornerstonejs/core'; +import type { Types } from '@cornerstonejs/core'; + +import { BaseTool } from '../base'; +import type { LabelmapSegmentationDataVolume } from '../../types/LabelmapTypes'; +import SegmentationRepresentations from '../../enums/SegmentationRepresentations'; +import type vtkImageData from '@kitware/vtk.js/Common/DataModel/ImageData'; +import { getActiveSegmentation } from '../../stateManagement/segmentation/getActiveSegmentation'; +import { getLockedSegmentIndices } from '../../stateManagement/segmentation/segmentLocking'; +import { getSegmentation } from '../../stateManagement/segmentation/getSegmentation'; +import { getClosestImageIdForStackViewport } from '../../utilities/annotationHydration'; +import { getCurrentLabelmapImageIdForViewport } from '../../stateManagement/segmentation/getCurrentLabelmapImageIdForViewport'; +import { getStackSegmentationImageIdsForViewport } from '../../stateManagement/segmentation/getStackSegmentationImageIdsForViewport'; +import { getSegmentIndexColor } from '../../stateManagement/segmentation/config/segmentationColor'; +import { getActiveSegmentIndex } from '../../stateManagement/segmentation/getActiveSegmentIndex'; +import { StrategyCallbacks } from '../../enums'; + +/** + * A type for preview data/information, used to setup previews on hover, or + * maintain the preview information. + */ +export type PreviewData = { + /** + * The preview data returned from the strategy + */ + preview: unknown; + /** A timer id to allow cancelling the timer */ + timer?: number; + /** The start time for the timer, to allow showing preview after a given length of time */ + timerStart: number; + /** + * The starting point where the use clicked down on, used to cancel preview + * on drag, but preserve it if the user moves the mouse tiny amounts accidentally. + */ + startPoint: Types.Point2; + element: HTMLDivElement; + /** + * Record if this is a drag preview, that is, a preview which is being extended + * by the user dragging to view more area. + */ + isDrag: boolean; +}; + +/** + * Labelmap tool containing shared functionality for labelmap tools. + */ +export default class LabelmapBaseTool extends BaseTool { + protected _editData: { + override: { + voxelManager: Types.IVoxelManager; + imageData: vtkImageData; + }; + segmentsLocked: number[]; // + imageId?: string; // stack labelmap + imageIds?: string[]; // stack labelmap + volumeId?: string; // volume labelmap + referencedVolumeId?: string; + } | null; + protected _hoverData?: { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + brushCursor: any; + segmentationId: string; + segmentIndex: number; + segmentColor: [number, number, number, number]; + viewportIdsToRender: string[]; + centerCanvas?: Array; + viewport: Types.IViewport; + }; + + protected _previewData?: PreviewData = { + preview: null, + element: null, + timerStart: 0, + timer: null, + startPoint: [NaN, NaN], + isDrag: false, + }; + + constructor(toolProps, defaultToolProps) { + super(toolProps, defaultToolProps); + } + + /** + * Creates a labelmap memo instance, which is a partially created memo + * object that stores the changes made to the labelmap rather than the + * initial state. This memo is then committed once done so that the + */ + public createMemo(segmentId: string, segmentationVoxelManager, preview) { + // TODO: Implement this + console.warn('LabelmapBaseTool.createMemo not implemented yet'); + // this.memo ||= LabelmapMemo.createLabelmapMemo( + // segmentId, + // segmentationVoxelManager, + // preview + // ); + // return this.memo as LabelmapMemo.LabelmapMemo; + } + + createEditData(element) { + const enabledElement = getEnabledElement(element); + const { viewport } = enabledElement; + + const activeSegmentation = getActiveSegmentation(viewport.id); + if (!activeSegmentation) { + const event = new CustomEvent(Enums.Events.ERROR_EVENT, { + detail: { + type: 'Segmentation', + message: + 'No active segmentation detected, create a segmentation representation before using the brush tool', + }, + cancelable: true, + }); + eventTarget.dispatchEvent(event); + return null; + } + + const { segmentationId } = activeSegmentation; + + const segmentsLocked = getLockedSegmentIndices(segmentationId); + + const { representationData } = getSegmentation(segmentationId); + + if (viewport instanceof BaseVolumeViewport) { + const { volumeId } = representationData[ + SegmentationRepresentations.Labelmap + ] as LabelmapSegmentationDataVolume; + const actors = viewport.getActors(); + + const isStackViewport = + viewport instanceof getClosestImageIdForStackViewport; + + if (isStackViewport) { + const event = new CustomEvent(Enums.Events.ERROR_EVENT, { + detail: { + type: 'Segmentation', + message: 'Cannot perform brush operation on the selected viewport', + }, + cancelable: true, + }); + eventTarget.dispatchEvent(event); + return null; + } + + // we used to take the first actor here but we should take the one that is + // probably the same size as the segmentation volume + const volumes = actors.map((actorEntry) => + cache.getVolume(actorEntry.referencedId) + ); + + const segmentationVolume = cache.getVolume(volumeId); + + const referencedVolumeIdToThreshold = + volumes.find((volume) => + csUtils.isEqual(volume.dimensions, segmentationVolume.dimensions) + )?.volumeId || volumes[0]?.volumeId; + + return { + volumeId, + referencedVolumeId: + this.configuration.thresholdVolumeId ?? referencedVolumeIdToThreshold, + segmentsLocked, + }; + } else { + const segmentationImageId = getCurrentLabelmapImageIdForViewport( + viewport.id, + segmentationId + ); + + if (!segmentationImageId) { + // if there is no stack segmentation slice for the current image + // we should not allow the user to perform any operation + return; + } + + // I hate this, but what can you do sometimes + if (this.configuration.activeStrategy.includes('SPHERE')) { + const referencedImageIds = viewport.getImageIds(); + const isValidVolumeForSphere = + csUtils.isValidVolume(referencedImageIds); + + if (!isValidVolumeForSphere) { + throw new Error( + 'Volume is not reconstructable for sphere manipulation' + ); + } + + const volumeId = `${segmentationId}_${viewport.id}`; + const volume = cache.getVolume(volumeId); + if (volume) { + return { + imageId: segmentationImageId, + segmentsLocked, + override: { + voxelManager: volume.voxelManager, + imageData: volume.imageData, + }, + }; + } else { + const labelmapImageIds = getStackSegmentationImageIdsForViewport( + viewport.id, + segmentationId + ); + + if (!labelmapImageIds || labelmapImageIds.length === 1) { + return { + imageId: segmentationImageId, + segmentsLocked, + }; + } + + // it will return the cached volume if it already exists + const volume = volumeLoader.createAndCacheVolumeFromImagesSync( + volumeId, + labelmapImageIds + ); + + return { + imageId: segmentationImageId, + segmentsLocked, + override: { + voxelManager: volume.voxelManager, + imageData: volume.imageData, + }, + }; + } + } else { + return { + imageId: segmentationImageId, + segmentsLocked, + }; + } + } + } + + protected createHoverData(element, centerCanvas?) { + const enabledElement = getEnabledElement(element); + const { viewport } = enabledElement; + + const camera = viewport.getCamera(); + const { viewPlaneNormal, viewUp } = camera; + + const viewportIdsToRender = [viewport.id]; + + const { segmentIndex, segmentationId, segmentColor } = + this.getActiveSegmentationData(viewport) || {}; + + // Center of circle in canvas Coordinates + const brushCursor = { + metadata: { + viewPlaneNormal: [...viewPlaneNormal], + viewUp: [...viewUp], + FrameOfReferenceUID: viewport.getFrameOfReferenceUID(), + referencedImageId: '', + toolName: this.getToolName(), + segmentColor, + }, + data: {}, + }; + + return { + brushCursor, + centerCanvas, + segmentIndex, + viewport, + segmentationId, + segmentColor, + viewportIdsToRender, + }; + } + + protected getActiveSegmentationData(viewport) { + const viewportId = viewport.id; + const activeRepresentation = getActiveSegmentation(viewportId); + + if (!activeRepresentation) { + return; + } + + const { segmentationId } = activeRepresentation; + const segmentIndex = getActiveSegmentIndex(segmentationId); + + if (!segmentIndex) { + return; + } + + const segmentColor = getSegmentIndexColor( + viewportId, + segmentationId, + segmentIndex + ); + + return { + segmentIndex, + segmentationId, + segmentColor, + }; + } + + protected getOperationData(element?) { + const editData = this._editData || this.createEditData(element); + const { segmentIndex, segmentationId, brushCursor } = + this._hoverData || this.createHoverData(element); + const { data, metadata = {} } = brushCursor || {}; + const { viewPlaneNormal, viewUp } = metadata; + const operationData = { + ...editData, + points: data?.handles?.points, + segmentIndex, + previewColors: + this.configuration.preview?.enabled || this._previewData.preview + ? this.configuration.preview.previewColors + : null, + viewPlaneNormal, + toolGroupId: this.toolGroupId, + segmentationId, + viewUp, + strategySpecificConfiguration: + this.configuration.strategySpecificConfiguration, + // Provide the preview information so that data can be used directly + preview: this._previewData?.preview, + }; + return operationData; + } + + /** + * Adds a preview that can be filled with data. + */ + public addPreview( + element = this._previewData.element, + options?: { acceptReject: boolean } + ) { + const acceptReject = options?.acceptReject; + if (acceptReject === true) { + this.acceptPreview(element); + } else if (acceptReject === false) { + this.rejectPreview(element); + } + const enabledElement = getEnabledElement(element); + this._previewData.preview = this.applyActiveStrategyCallback( + enabledElement, + this.getOperationData(element), + StrategyCallbacks.AddPreview + ); + this._previewData.isDrag = true; + return this._previewData.preview; + } + + /** + * Cancels any preview view being shown, resetting any segments being shown. + */ + public rejectPreview(element = this._previewData.element) { + if (!element || !this._previewData.preview) { + return; + } + const enabledElement = getEnabledElement(element); + this.applyActiveStrategyCallback( + enabledElement, + this.getOperationData(element), + StrategyCallbacks.RejectPreview + ); + this._previewData.preview = null; + this._previewData.isDrag = false; + } + + /** + * Accepts a preview, marking it as the active segment. + */ + public acceptPreview(element = this._previewData.element) { + if (!element) { + return; + } + const enabledElement = getEnabledElement(element); + + this.applyActiveStrategyCallback( + enabledElement, + this.getOperationData(element), + StrategyCallbacks.AcceptPreview + ); + this._previewData.isDrag = false; + this._previewData.preview = null; + } +} diff --git a/packages/tools/src/tools/segmentation/strategies/BrushStrategy.ts b/packages/tools/src/tools/segmentation/strategies/BrushStrategy.ts index 2c4ca44e67..713c0981ec 100644 --- a/packages/tools/src/tools/segmentation/strategies/BrushStrategy.ts +++ b/packages/tools/src/tools/segmentation/strategies/BrushStrategy.ts @@ -113,6 +113,7 @@ export default class BrushStrategy { [StrategyCallbacks.ComputeInnerCircleRadius]: addListMethod( StrategyCallbacks.ComputeInnerCircleRadius ), + [StrategyCallbacks.AddPreview]: addListMethod(StrategyCallbacks.AddPreview), [StrategyCallbacks.GetStatistics]: addSingletonMethod( StrategyCallbacks.GetStatistics ), @@ -311,6 +312,28 @@ export default class BrushStrategy { operationData: LabelmapToolOperationDataAny ) => void; + /** + * Adds a preview to the view, without filling it with any contents, returning + * the initialized preview data. + */ + public addPreview = ( + enabledElement, + operationData: LabelmapToolOperationDataAny + ) => { + const initializedData = this.initialize( + enabledElement, + operationData, + StrategyCallbacks.AddPreview + ); + + if (!initializedData) { + // Happens when there is no label map + return; + } + + return initializedData.preview || initializedData; + }; + /** * Accept the preview, making it part of the overall segmentation * diff --git a/packages/tools/src/tools/segmentation/strategies/compositions/determineSegmentIndex.ts b/packages/tools/src/tools/segmentation/strategies/compositions/determineSegmentIndex.ts index 70674aef1b..148181e628 100644 --- a/packages/tools/src/tools/segmentation/strategies/compositions/determineSegmentIndex.ts +++ b/packages/tools/src/tools/segmentation/strategies/compositions/determineSegmentIndex.ts @@ -1,5 +1,6 @@ import type { InitializedOperationData } from '../BrushStrategy'; import StrategyCallbacks from '../../../../enums/StrategyCallbacks'; +import type { Types } from '@cornerstonejs/core'; /** * This function determines whether to fill or erase based on what the user @@ -38,7 +39,7 @@ export default { segmentationVoxelManager, centerIJK, strategySpecificConfiguration, - imageVoxelManager, + viewPlaneNormal, segmentationImageData, preview, } = operationData; @@ -50,6 +51,19 @@ export default { let hasSegmentIndex = false; let hasPreviewIndex = false; + + const nestedBounds = [ + ...segmentationVoxelManager.getBoundsIJK(), + ]; + + if (Math.abs(viewPlaneNormal[0]) > 0.8) { + nestedBounds[0] = [centerIJK[0], centerIJK[0]]; + } else if (Math.abs(viewPlaneNormal[1]) > 0.8) { + nestedBounds[1] = [centerIJK[1], centerIJK[1]]; + } else if (Math.abs(viewPlaneNormal[2]) > 0.8) { + nestedBounds[2] = [centerIJK[2], centerIJK[2]]; + } + const callback = ({ value }) => { hasSegmentIndex ||= value === segmentIndex; hasPreviewIndex ||= value === previewSegmentIndex; @@ -58,6 +72,7 @@ export default { segmentationVoxelManager.forEach(callback, { imageData: segmentationImageData, isInObject: operationData.isInObject, + boundsIJK: nestedBounds, }); if (!hasSegmentIndex && !hasPreviewIndex) { diff --git a/packages/tools/src/types/AnnotationTypes.ts b/packages/tools/src/types/AnnotationTypes.ts index bb156818a0..87d16318a9 100644 --- a/packages/tools/src/types/AnnotationTypes.ts +++ b/packages/tools/src/types/AnnotationTypes.ts @@ -10,6 +10,10 @@ type Annotation = { * hole inside a contour. */ parentAnnotationUID?: string; + /** + * The interpolationUID, to match up annotations getting interpolated + */ + interpolationUID?: string; /** * Array that contains all child annotation UID * @@ -76,7 +80,7 @@ type Annotation = { }; [key: string]: unknown; /** Cached Annotation statistics which is specific to the tool */ - cachedStats?: unknown; + cachedStats?: Record; }; }; diff --git a/utils/ExampleRunner/build-all-examples-cli.js b/utils/ExampleRunner/build-all-examples-cli.js index bcb31b8628..977b7b87e5 100644 --- a/utils/ExampleRunner/build-all-examples-cli.js +++ b/utils/ExampleRunner/build-all-examples-cli.js @@ -55,6 +55,7 @@ if (options.fromRoot === true) { examples: [ { path: 'packages/core/examples', regexp: 'index.ts' }, { path: 'packages/tools/examples', regexp: 'index.ts' }, + { path: 'packages/ai/examples', regexp: 'index.ts' }, { path: 'packages/dicomImageLoader/examples', regexp: 'index.ts', diff --git a/utils/ExampleRunner/example-info.json b/utils/ExampleRunner/example-info.json index 31cd2b933e..bfd5bd2152 100644 --- a/utils/ExampleRunner/example-info.json +++ b/utils/ExampleRunner/example-info.json @@ -407,6 +407,10 @@ "sculptorTool": { "name": "sculptorTool Tool", "description": "Demonstrates how to have similar brush tool effects on the contour" + }, + "segmentationAI": { + "name": "Segmentation AI Assistance", + "description": "Demonstrates how to use AI/ML assistance tools for segmentation creation" } }, "tools-advanced": { diff --git a/utils/ExampleRunner/example-runner-cli.js b/utils/ExampleRunner/example-runner-cli.js index 9276a13f11..f406b556b6 100755 --- a/utils/ExampleRunner/example-runner-cli.js +++ b/utils/ExampleRunner/example-runner-cli.js @@ -106,14 +106,11 @@ function levenshteinDistance(a, b) { return matrix[b.length][a.length]; } -// ---------------------------------------------------------------------------- -// Find examples -// ---------------------------------------------------------------------------- - const configuration = { examples: [ { path: 'packages/core/examples', regexp: 'index.ts' }, { path: 'packages/tools/examples', regexp: 'index.ts' }, + { path: 'packages/ai/examples', regexp: 'index.ts' }, { path: 'packages/dicomImageLoader/examples', regexp: 'index.ts', diff --git a/utils/ExampleRunner/template-config.js b/utils/ExampleRunner/template-config.js index 7c08ae42a1..032ad27db5 100644 --- a/utils/ExampleRunner/template-config.js +++ b/utils/ExampleRunner/template-config.js @@ -2,6 +2,7 @@ const path = require('path'); const csRenderBasePath = path.resolve('packages/core/src/index'); const csToolsBasePath = path.resolve('packages/tools/src/index'); +const csAiBasePath = path.resolve('packages/ai/src/index'); const csAdapters = path.resolve('packages/adapters/src/index'); const csDICOMImageLoaderDistPath = path.resolve( 'packages/dicomImageLoader/src/index' @@ -36,6 +37,11 @@ module.exports = { to: '${destPath.replace(/\\/g, '/')}', noErrorOnMissing: true, }, + { + from: + '../../../node_modules/onnxruntime-web/dist', + to: '${destPath.replace(/\\/g, '/')}/dist', + }, ], }), ], @@ -60,6 +66,7 @@ module.exports = { alias: { '@cornerstonejs/core': '${csRenderBasePath.replace(/\\/g, '/')}', '@cornerstonejs/tools': '${csToolsBasePath.replace(/\\/g, '/')}', + '@cornerstonejs/ai': '${csAiBasePath.replace(/\\/g, '/')}', '@cornerstonejs/nifti-volume-loader': '${csNiftiPath.replace( /\\/g, '/' diff --git a/utils/ExampleRunner/template-multiexample-config.js b/utils/ExampleRunner/template-multiexample-config.js index c24912e7d9..7f84f0028e 100644 --- a/utils/ExampleRunner/template-multiexample-config.js +++ b/utils/ExampleRunner/template-multiexample-config.js @@ -2,6 +2,7 @@ const path = require('path'); const csRenderBasePath = path.resolve('./packages/core/src/index'); const csToolsBasePath = path.resolve('./packages/tools/src/index'); +const csAiBasePath = path.resolve('./packages/ai/src/index'); const csAdaptersBasePath = path.resolve('./packages/adapters/src/index'); const csDICOMImageLoaderDistPath = path.resolve( 'packages/dicomImageLoader/src/index' @@ -69,6 +70,11 @@ module.exports = { to: '${destPath.replace(/\\/g, '/')}', noErrorOnMissing: true, }, + { + from: + '../../../node_modules/onnxruntime-web/dist', + to: '${destPath.replace(/\\/g, '/')}/dist', + }, ], }), ], @@ -95,6 +101,7 @@ module.exports = { alias: { '@cornerstonejs/core': '${csRenderBasePath.replace(/\\/g, '/')}', '@cornerstonejs/tools': '${csToolsBasePath.replace(/\\/g, '/')}', + '@cornerstonejs/ai': '${csAiBasePath.replace(/\\/g, '/')}', '@cornerstonejs/adapters': '${csAdaptersBasePath.replace(/\\/g, '/')}', '@cornerstonejs/dicom-image-loader': '${csDICOMImageLoaderDistPath.replace( /\\/g, diff --git a/utils/demo/helpers/labelmapTools.ts b/utils/demo/helpers/labelmapTools.ts index 9c00519448..287bd431f4 100644 --- a/utils/demo/helpers/labelmapTools.ts +++ b/utils/demo/helpers/labelmapTools.ts @@ -13,9 +13,10 @@ const previewColors = { 1: [0, 255, 255, 255], }; const preview = { - enabled: false, + enabled: true, previewColors, }; + const configuration = { preview, strategySpecificConfiguration: { @@ -23,6 +24,13 @@ const configuration = { }, }; +const configurationNoPreview = { + preview: { enabled: false, previewColors }, + strategySpecificConfiguration: { + useCenterSegmentIndex: false, + }, +}; + const thresholdOptions = new Map(); thresholdOptions.set('Dynamic Radius 0', { isDynamic: true, dynamicRadius: 0 }); thresholdOptions.set('Dynamic Radius 1', { isDynamic: true, dynamicRadius: 1 }); @@ -45,20 +53,24 @@ const defaultThresholdOption = [...thresholdOptions.keys()][2]; const thresholdArgs = thresholdOptions.get(defaultThresholdOption); const toolMap = new Map(); -toolMap.set('SphereBrush', { +toolMap.set('ThresholdCircle', { + tool: BrushTool, baseTool: BrushTool.toolName, configuration: { ...configuration, - activeStrategy: 'FILL_INSIDE_SPHERE', + activeStrategy: 'THRESHOLD_INSIDE_CIRCLE', + strategySpecificConfiguration: { + ...configuration.strategySpecificConfiguration, + THRESHOLD: { ...thresholdArgs }, + }, }, }); -toolMap.set('ThresholdCircle', { - tool: BrushTool, +toolMap.set('ThresholdSphere', { baseTool: BrushTool.toolName, configuration: { ...configuration, - activeStrategy: 'THRESHOLD_INSIDE_CIRCLE', + activeStrategy: 'THRESHOLD_INSIDE_SPHERE_WITH_ISLAND_REMOVAL', strategySpecificConfiguration: { ...configuration.strategySpecificConfiguration, THRESHOLD: { ...thresholdArgs }, @@ -69,7 +81,7 @@ toolMap.set('ThresholdCircle', { toolMap.set('CircularBrush', { baseTool: BrushTool.toolName, configuration: { - ...configuration, + ...configurationNoPreview, activeStrategy: 'FILL_INSIDE_CIRCLE', }, }); @@ -77,29 +89,36 @@ toolMap.set('CircularBrush', { toolMap.set('CircularEraser', { baseTool: BrushTool.toolName, configuration: { - ...configuration, + ...configurationNoPreview, activeStrategy: 'ERASE_INSIDE_CIRCLE', }, }); +toolMap.set('SphereBrush', { + baseTool: BrushTool.toolName, + configuration: { + ...configurationNoPreview, + activeStrategy: 'FILL_INSIDE_SPHERE', + }, +}); toolMap.set('SphereEraser', { baseTool: BrushTool.toolName, configuration: { - ...configuration, + ...configurationNoPreview, activeStrategy: 'ERASE_INSIDE_SPHERE', }, }); toolMap.set(RectangleScissorsTool.toolName, { tool: RectangleScissorsTool }); toolMap.set(CircleScissorsTool.toolName, { tool: CircleScissorsTool }); toolMap.set(SphereScissorsTool.toolName, { tool: SphereScissorsTool }); -toolMap.set('ScissorsEraser', { +toolMap.set('SphereScissorsEraser', { baseTool: SphereScissorsTool.toolName, configuration: { - ...configuration, + ...configurationNoPreview, activeStrategy: 'ERASE_INSIDE', }, }); -toolMap.set(PaintFillTool.toolName, {}); +toolMap.set(PaintFillTool.toolName, { tool: PaintFillTool }); const labelmapTools = { toolMap, diff --git a/yarn.lock b/yarn.lock index 852d9c19fa..c051a8e44c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5072,6 +5072,59 @@ resolved "https://registry.yarnpkg.com/@polka/url/-/url-1.0.0-next.25.tgz#f077fdc0b5d0078d30893396ff4827a13f99e817" integrity sha512-j7P6Rgr3mmtdkeDGTe0E/aYyWEWVtc5yFXtHCRHs28/jptDEWfaVOc5T7cblqy1XKPPfCxJc/8DwQ5YgLOZOVQ== +"@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/aspromise/-/aspromise-1.1.2.tgz#9b8b0cc663d669a7d8f6f5d0893a14d348f30fbf" + integrity sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ== + +"@protobufjs/base64@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/base64/-/base64-1.1.2.tgz#4c85730e59b9a1f1f349047dbf24296034bb2735" + integrity sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg== + +"@protobufjs/codegen@^2.0.4": + version "2.0.4" + resolved "https://registry.yarnpkg.com/@protobufjs/codegen/-/codegen-2.0.4.tgz#7ef37f0d010fb028ad1ad59722e506d9262815cb" + integrity sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg== + +"@protobufjs/eventemitter@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz#355cbc98bafad5978f9ed095f397621f1d066b70" + integrity sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q== + +"@protobufjs/fetch@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/fetch/-/fetch-1.1.0.tgz#ba99fb598614af65700c1619ff06d454b0d84c45" + integrity sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ== + dependencies: + "@protobufjs/aspromise" "^1.1.1" + "@protobufjs/inquire" "^1.1.0" + +"@protobufjs/float@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@protobufjs/float/-/float-1.0.2.tgz#5e9e1abdcb73fc0a7cb8b291df78c8cbd97b87d1" + integrity sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ== + +"@protobufjs/inquire@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/inquire/-/inquire-1.1.0.tgz#ff200e3e7cf2429e2dcafc1140828e8cc638f089" + integrity sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q== + +"@protobufjs/path@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/path/-/path-1.1.2.tgz#6cc2b20c5c9ad6ad0dccfd21ca7673d8d7fbf68d" + integrity sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA== + +"@protobufjs/pool@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/pool/-/pool-1.1.0.tgz#09fd15f2d6d3abfa9b65bc366506d6ad7846ff54" + integrity sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw== + +"@protobufjs/utf8@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" + integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== + "@remix-run/router@1.16.1": version "1.16.1" resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.16.1.tgz#73db3c48b975eeb06d0006481bde4f5f2d17d1cd" @@ -6194,6 +6247,13 @@ dependencies: undici-types "~6.18.2" +"@types/node@>=13.7.0": + version "22.10.1" + resolved "https://registry.yarnpkg.com/@types/node/-/node-22.10.1.tgz#41ffeee127b8975a05f8c4f83fb89bcb2987d766" + integrity sha512-qKgsUwfHZV2WCWLAnVP1JqnpE6Im6h3Y0+fYgMTasNQ7V++CBX5OT1as0g0f+OyubbFqhf6XVNIsmN4IIhEgGQ== + dependencies: + undici-types "~6.20.0" + "@types/node@^17.0.5": version "17.0.45" resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.45.tgz#2c0fafd78705e7a18b7906b5201a522719dc5190" @@ -11779,6 +11839,11 @@ flat@^5.0.2: resolved "https://registry.yarnpkg.com/flat/-/flat-5.0.2.tgz#8ca6fe332069ffa9d324c327198c598259ceb241" integrity sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ== +flatbuffers@^1.12.0: + version "1.12.0" + resolved "https://registry.yarnpkg.com/flatbuffers/-/flatbuffers-1.12.0.tgz#72e87d1726cb1b216e839ef02658aa87dcef68aa" + integrity sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ== + flatted@^3.2.7, flatted@^3.2.9: version "3.3.1" resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.3.1.tgz#21db470729a6734d4997002f439cb308987f567a" @@ -12536,6 +12601,11 @@ gray-matter@^4.0.3: section-matter "^1.0.0" strip-bom-string "^1.0.0" +guid-typescript@^1.0.9: + version "1.0.9" + resolved "https://registry.yarnpkg.com/guid-typescript/-/guid-typescript-1.0.9.tgz#e35f77003535b0297ea08548f5ace6adb1480ddc" + integrity sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ== + gzip-size@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/gzip-size/-/gzip-size-6.0.0.tgz#065367fd50c239c0671cbcbad5be3e2eeb10e462" @@ -15593,6 +15663,11 @@ loglevelnext@^3.0.1: resolved "https://registry.yarnpkg.com/loglevelnext/-/loglevelnext-3.0.1.tgz#e3e4659c4061c09264f6812c33586dc55a009a04" integrity sha512-JpjaJhIN1reaSb26SIxDGtE0uc67gPl19OMVHrr+Ggt6b/Vy60jmCtKgQBrygAH0bhRA2nkxgDvM+8QvR8r0YA== +long@^5.0.0, long@^5.2.3: + version "5.2.3" + resolved "https://registry.yarnpkg.com/long/-/long-5.2.3.tgz#a3ba97f3877cf1d778eccbcb048525ebb77499e1" + integrity sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q== + longest-streak@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/longest-streak/-/longest-streak-3.1.0.tgz#62fa67cd958742a1574af9f39866364102d90cd4" @@ -17849,6 +17924,23 @@ oniguruma-to-js@0.4.3: dependencies: regex "^4.3.2" +onnxruntime-common@1.17.1: + version "1.17.1" + resolved "https://registry.yarnpkg.com/onnxruntime-common/-/onnxruntime-common-1.17.1.tgz#1d3cfcfe56f9a5fc2ef5530c8e8723c86708a290" + integrity sha512-6wLNhpn+1hnsKN+jq6ulqUEJ61TdRmyFkGCvtRNnZkAupH8Yfr805UeNxjl9jtiX9B1q48pq6Q/67fEFpxT7Dw== + +onnxruntime-web@1.17.1: + version "1.17.1" + resolved "https://registry.yarnpkg.com/onnxruntime-web/-/onnxruntime-web-1.17.1.tgz#ff0c5ecdcc4063e653f3cac4ff8e049a57301f60" + integrity sha512-EotY9uJU4xFY/ZVZ2Zrl2OZmBcbTVTWn/2OOh4cCWODPwtsYN2xeJYgoz8LfCgZSrhenGg0q4ceYUWATXqEsYQ== + dependencies: + flatbuffers "^1.12.0" + guid-typescript "^1.0.9" + long "^5.2.3" + onnxruntime-common "1.17.1" + platform "^1.3.6" + protobufjs "^7.2.4" + open-cli@^7.0.1: version "7.2.0" resolved "https://registry.yarnpkg.com/open-cli/-/open-cli-7.2.0.tgz#9431203847648890026c54c08dcd3430c6fce23f" @@ -18653,6 +18745,11 @@ pkg-up@^3.1.0: dependencies: find-up "^3.0.0" +platform@^1.3.6: + version "1.3.6" + resolved "https://registry.yarnpkg.com/platform/-/platform-1.3.6.tgz#48b4ce983164b209c2d45a107adb31f473a6e7a7" + integrity sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg== + playwright-core@1.48.1: version "1.48.1" resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.48.1.tgz#5fe28fb9a9326dae88d4608c35e819163cceeb23" @@ -19455,6 +19552,24 @@ proto-list@~1.2.1: resolved "https://registry.yarnpkg.com/proto-list/-/proto-list-1.2.4.tgz#212d5bfe1318306a420f6402b8e26ff39647a849" integrity sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA== +protobufjs@^7.2.4: + version "7.4.0" + resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.4.0.tgz#7efe324ce9b3b61c82aae5de810d287bc08a248a" + integrity sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw== + dependencies: + "@protobufjs/aspromise" "^1.1.2" + "@protobufjs/base64" "^1.1.2" + "@protobufjs/codegen" "^2.0.4" + "@protobufjs/eventemitter" "^1.1.0" + "@protobufjs/fetch" "^1.1.0" + "@protobufjs/float" "^1.0.2" + "@protobufjs/inquire" "^1.1.0" + "@protobufjs/path" "^1.1.2" + "@protobufjs/pool" "^1.1.0" + "@protobufjs/utf8" "^1.1.0" + "@types/node" ">=13.7.0" + long "^5.0.0" + protocol-buffers-schema@^3.3.1: version "3.6.0" resolved "https://registry.yarnpkg.com/protocol-buffers-schema/-/protocol-buffers-schema-3.6.0.tgz#77bc75a48b2ff142c1ad5b5b90c94cd0fa2efd03" @@ -22694,6 +22809,11 @@ undici-types@~6.18.2: resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.18.2.tgz#8b678cf939d4fc9ec56be3c68ed69c619dee28b0" integrity sha512-5ruQbENj95yDYJNS3TvcaxPMshV7aizdv/hWYjGIKoANWKjhWNBsr2YEuYZKodQulB1b8l7ILOuDQep3afowQQ== +undici-types@~6.20.0: + version "6.20.0" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.20.0.tgz#8171bf22c1f588d1554d55bf204bc624af388433" + integrity sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg== + unenv@^1.9.0: version "1.10.0" resolved "https://registry.yarnpkg.com/unenv/-/unenv-1.10.0.tgz#c3394a6c6e4cfe68d699f87af456fe3f0db39571"