Add MONAI Bundle inference operator and update example app (#303)

* WIP MONAI Bundle inference operator.

Signed-off-by: mmelqin <mingmelvinq@nvidia.com>

* Further simplified the code requiring model name only for bundle.

Signed-off-by: mmelqin <mingmelvinq@nvidia.com>

* Check in the bundle operator and multipl model support.

Signed-off-by: mmelqin <mingmelvinq@nvidia.com>

* Added changes for bundle operator and app.

Signed-off-by: mmelqin <mingmelvinq@nvidia.com>

* Fix styling check errors.

Signed-off-by: mmelqin <mingmelvinq@nvidia.com>

* Fix flake8 check error.

Signed-off-by: mmelqin <mingmelvinq@nvidia.com>

* Correct all MyPy type checking errors and improve inline doc.

The base inference operator needs to adjust the function signatures due to the expanded
use of types resulting from the intro if bundle inference operator.
The new version of MyPy also seems to be stricter on types.

Signed-off-by: mmelqin <mingmelvinq@nvidia.com>

* MyPy on Git is complaining things that are not found in local checking.

Signed-off-by: mmelqin <mingmelvinq@nvidia.com>

* Update per review comments.

Signed-off-by: mmelqin <mingmelvinq@nvidia.com>

* Rebased for the requirements.txt file. Not req'ed for docs/requirements but for consistency.

Signed-off-by: mmelqin <mingmelvinq@nvidia.com>

* Add the notebook for creating app with MONAI Bundle inference operator.

Signed-off-by: mmelqin <mingmelvinq@nvidia.com>

* Updating the notebook descriptions.

Signed-off-by: mmelqin <mingmelvinq@nvidia.com>

* Update and make consistent the user guides

Signed-off-by: mmelqin <mingmelvinq@nvidia.com>

Author: MMelQin

docs/requirements.txt

@@ -1,7 +1,7 @@
 Sphinx==4.1.2
 sphinx-autobuild==2021.3.14
 myst-parser==0.15.2
-numpy==1.21  # CVE-2021-33430
+numpy==1.21.2  # CVE-2021-33430
 matplotlib==3.3.4
 ipywidgets==7.6.4
 pandas==1.1.5
@@ -19,10 +19,13 @@ docutils==0.16  # 0.17 causes error. https://github.com/executablebooks/MyST-Par
 pydata_sphinx_theme==0.6.3
 sphinxemoji==0.1.8
 scipy
-scikit-image
+scikit-image>=0.17.2
 plotly
-nibabel
-monai
+nibabel>=3.2.1
+monai>=0.9.0
+torch>=1.10.0
+numpy-stl>=2.12.0
+trimesh>=3.8.11
 pydicom
 sphinx-autodoc-typehints==1.12.0
 sphinxcontrib-applehelp==1.0.2
@@ -31,4 +34,4 @@ sphinxcontrib-htmlhelp==2.0.0
 sphinxcontrib-jsmath==1.0.1
 sphinxcontrib-qthelp==1.0.3
 sphinxcontrib-serializinghtml==1.1.5
-sphinxcontrib-mermaid==0.7.1
+sphinxcontrib-mermaid==0.7.1

docs/source/getting_started/examples.md

@@ -5,7 +5,9 @@
 <https://github.com/Project-MONAI/monai-deploy-app-sdk/tree/main/examples/apps> has example apps that you can see.
 
 - ai_spleen_seg_app
+- ai_livertumor_seg_app
 - ai_unetr_seg_app
 - dicom_series_to_image_app
 - mednist_classifier_monaideploy
 - simple_imaging_app
+- deply_app_on_aarch64

docs/source/getting_started/tutorials/03_segmentation_app.md

@@ -15,7 +15,7 @@ jupyter-lab
 ```
 
 ## Executing from Jupyter Notebook
-
+Please note that the example code used in the Jupyter Notebook is based on an earlier version of the segmentation application, hence not the same as the latest source code on Github, e.g. not using MONAI Bundle inference operator.
 ```{toctree}
 :maxdepth: 4
 
@@ -43,7 +43,7 @@ jupyter-lab
 ```
 
 ## Executing from Shell
-
+Please note that this part of the example uses the latest application source code on Github, as well as the corresponding test data.
 ```bash
 # Clone the github project (the latest version of main branch only)
 git clone --branch main --depth 1 https://github.com/Project-MONAI/monai-deploy-app-sdk.git
@@ -53,17 +53,18 @@ cd monai-deploy-app-sdk
 # Install monai-deploy-app-sdk package
 pip install monai-deploy-app-sdk
 
-# Download/Extract ai_spleen_seg_data zip file from https://drive.google.com/file/d/1GC_N8YQk_mOWN02oOzAU_2YDmNRWk--n/view?usp=sharing
+# Download/Extract ai_spleen_bundle_data zip file from https://drive.google.com/file/d/1cJq0iQh_yzYIxVElSlVa141aEmHZADJh/view?usp=sharing
 
-# Download ai_spleen_seg_data.zip
+# Download ai_spleen_bundle_data.zip
 pip install gdown
-gdown https://drive.google.com/uc?id=1GC_N8YQk_mOWN02oOzAU_2YDmNRWk--n
+gdown https://drive.google.com/uc?id=1cJq0iQh_yzYIxVElSlVa141aEmHZADJh
 
-# After downloading ai_spleen_seg_data.zip from the web browser or using gdown,
-unzip -o ai_spleen_seg_data_updated_1203.zip
+# After downloading ai_spleen_bundle_data.zip from the web browser or using gdown,
+unzip -o ai_spleen_bundle_data.zip
 
-# Install necessary packages from the app
-pip install monai pydicom SimpleITK Pillow nibabel
+# Install necessary packages from the app; note that numpy-stl and trimesh are only
+# needed if the application uses the STL Conversion Operator
+pip install monai pydicom SimpleITK Pillow nibabel scikit-image numpy-stl trimesh
 
 # Local execution of the app
 python examples/apps/ai_spleen_seg_app/app.py -i dcm/ -o output -m model.ts

docs/source/getting_started/tutorials/index.md

@@ -9,4 +9,5 @@
 03_segmentation_app
 04_mis_tutorial
 05_full_tutorial
+06_monai_bundle_app
 ```

examples/apps/ai_livertumor_seg_app/livertumor_seg_operator.py

@@ -94,6 +94,7 @@ def compute(self, op_input: InputContext, op_output: OutputContext, context: Exe
             pre_transforms,
             post_transforms,
             overlap=0.6,
+            model_name="",
         )
 
         # Setting the keys used in the dictironary based transforms may change.

examples/apps/ai_spleen_seg_app/app.py

@@ -1,4 +1,4 @@
-# Copyright 2021 MONAI Consortium
+# Copyright 2021-2022 MONAI Consortium
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -11,14 +11,16 @@
 
 import logging
 
-from spleen_seg_operator import SpleenSegOperator
-
 from monai.deploy.core import Application, resource
+from monai.deploy.core.domain import Image
+from monai.deploy.core.io_type import IOType
 from monai.deploy.operators.dicom_data_loader_operator import DICOMDataLoaderOperator
 from monai.deploy.operators.dicom_seg_writer_operator import DICOMSegmentationWriterOperator
 from monai.deploy.operators.dicom_series_selector_operator import DICOMSeriesSelectorOperator
 from monai.deploy.operators.dicom_series_to_volume_operator import DICOMSeriesToVolumeOperator
-from monai.deploy.operators.stl_conversion_operator import STLConversionOperator
+from monai.deploy.operators.monai_bundle_inference_operator import IOMapping, MonaiBundleInferenceOperator
+
+# from monai.deploy.operators.stl_conversion_operator import STLConversionOperator  # import as needed.
 
 
 @resource(cpu=1, gpu=1, memory="7Gi")
@@ -32,42 +34,54 @@ def __init__(self, *args, **kwargs):
 
     def run(self, *args, **kwargs):
         # This method calls the base class to run. Can be omitted if simply calling through.
-        self._logger.debug(f"Begin {self.run.__name__}")
+        self._logger.info(f"Begin {self.run.__name__}")
         super().run(*args, **kwargs)
-        self._logger.debug(f"End {self.run.__name__}")
+        self._logger.info(f"End {self.run.__name__}")
 
     def compose(self):
         """Creates the app specific operators and chain them up in the processing DAG."""
 
-        self._logger.debug(f"Begin {self.compose.__name__}")
+        logging.info(f"Begin {self.compose.__name__}")
 
-        # Creates the custom operator(s) as well as SDK built-in operator(s).
+        # Create the custom operator(s) as well as SDK built-in operator(s).
         study_loader_op = DICOMDataLoaderOperator()
         series_selector_op = DICOMSeriesSelectorOperator(Sample_Rules_Text)
         series_to_vol_op = DICOMSeriesToVolumeOperator()
-        # Model specific inference operator, supporting MONAI transforms.
-        spleen_seg_op = SpleenSegOperator()
+
+        # Create the inference operator that supports MONAI Bundle and automates the inference.
+        # The IOMapping labels match the input and prediction keys in the pre and post processing.
+        # The model_name is optional when the app has only one model.
+        # The bundle_path argument optionally can be set to an accessible bundle file path in the dev
+        # environment, so when the app is packaged into a MAP, the operator can complete the bundle parsing
+        # during init to provide the optional packages info, parsed from the bundle, to the packager
+        # for it to install the packages in the MAP docker image.
+        # Setting output IOType to DISK only works only for leaf operators, not the case in this example.
+        bundle_spleen_seg_op = MonaiBundleInferenceOperator(
+            input_mapping=[IOMapping("image", Image, IOType.IN_MEMORY)],
+            output_mapping=[IOMapping("pred", Image, IOType.IN_MEMORY)],
+        )
+
         # Create DICOM Seg writer with segment label name in a string list
         dicom_seg_writer = DICOMSegmentationWriterOperator(seg_labels=["Spleen"])
-        # Create the surface mesh STL conversion operator
-        stl_conversion_op = STLConversionOperator(output_file="stl/spleen.stl")
 
         # Create the processing pipeline, by specifying the upstream and downstream operators, and
         # ensuring the output from the former matches the input of the latter, in both name and type.
         self.add_flow(study_loader_op, series_selector_op, {"dicom_study_list": "dicom_study_list"})
         self.add_flow(
             series_selector_op, series_to_vol_op, {"study_selected_series_list": "study_selected_series_list"}
         )
-        self.add_flow(series_to_vol_op, spleen_seg_op, {"image": "image"})
+        self.add_flow(series_to_vol_op, bundle_spleen_seg_op, {"image": "image"})
         # Note below the dicom_seg_writer requires two inputs, each coming from a upstream operator.
         self.add_flow(
             series_selector_op, dicom_seg_writer, {"study_selected_series_list": "study_selected_series_list"}
         )
-        self.add_flow(spleen_seg_op, dicom_seg_writer, {"seg_image": "seg_image"})
-        # Add the STL conversion operator as another leaf operator taking as input the seg image.
-        self.add_flow(spleen_seg_op, stl_conversion_op, {"seg_image": "image"})
+        self.add_flow(bundle_spleen_seg_op, dicom_seg_writer, {"pred": "seg_image"})
+        # Create the surface mesh STL conversion operator and add it to the app execution flow, if needed, by
+        # uncommenting the following couple lines.
+        # stl_conversion_op = STLConversionOperator(output_file="stl/spleen.stl")
+        # self.add_flow(bundle_spleen_seg_op, stl_conversion_op, {"pred": "image"})
 
-        self._logger.debug(f"End {self.compose.__name__}")
+        logging.info(f"End {self.compose.__name__}")
 
 
 # This is a sample series selection rule in JSON, simply selecting CT series.
@@ -94,8 +108,7 @@ def compose(self):
     #     -i <DICOM folder>, for input DICOM CT series folder
     #     -o <output folder>, for the output folder, default $PWD/output
     # e.g.
-    #     python3 app.py -i input -m model/model.ts
+    #     monai-deploy exec app.py -i input -m model/model.ts
     #
     logging.basicConfig(level=logging.DEBUG)
-    app_instance = AISpleenSegApp()  # Optional params' defaults are fine.
-    app_instance.run()
+    app_instance = AISpleenSegApp(do_run=True)

examples/apps/ai_spleen_seg_app/spleen_seg_operator.py

@@ -12,13 +12,16 @@
 .. autosummary::
     :toctree: _autosummary
 
+    BundleConfigNames
     ClaraVizOperator
     DICOMDataLoaderOperator
     DICOMSegmentationWriterOperator
     DICOMSeriesSelectorOperator
     DICOMSeriesToVolumeOperator
     DICOMTextSRWriterOperator
     InferenceOperator
+    IOMapping
+    MonaiBundleInferenceOperator
     MonaiSegInferenceOperator
     PNGConverterOperator
     PublisherOperator
@@ -33,6 +36,7 @@
 from .dicom_series_to_volume_operator import DICOMSeriesToVolumeOperator
 from .dicom_text_sr_writer_operator import DICOMTextSRWriterOperator, EquipmentInfo, ModelInfo
 from .inference_operator import InferenceOperator
+from .monai_bundle_inference_operator import BundleConfigNames, IOMapping, MonaiBundleInferenceOperator
 from .monai_seg_inference_operator import MonaiSegInferenceOperator
 from .png_converter_operator import PNGConverterOperator
 from .publisher_operator import PublisherOperator

monai/deploy/operators/__init__.py

@@ -10,7 +10,7 @@
 # limitations under the License.
 
 from abc import abstractmethod
-from typing import Any, Union
+from typing import Any, Dict, Tuple, Union
 
 from monai.deploy.core import ExecutionContext, Image, InputContext, Operator, OutputContext
 
@@ -27,7 +27,7 @@ def __init__(self, *args, **kwargs):
         super().__init__()
 
     @abstractmethod
-    def pre_process(self, data: Any) -> Union[Image, Any]:
+    def pre_process(self, data: Any, *args, **kwargs) -> Union[Image, Any, Tuple[Any, ...], Dict[Any, Any]]:
         """Transforms input before being used for predicting on a model.
 
         This method must be overridden by a derived class.
@@ -50,7 +50,7 @@ def compute(self, op_input: InputContext, op_output: OutputContext, context: Exe
         pass
 
     @abstractmethod
-    def predict(self, data: Any) -> Union[Image, Any]:
+    def predict(self, data: Any, *args, **kwargs) -> Union[Image, Any, Tuple[Any, ...], Dict[Any, Any]]:
         """Predicts results using the models(s) with input tensors.
 
         This method must be overridden by a derived class.
@@ -61,7 +61,7 @@ def predict(self, data: Any) -> Union[Image, Any]:
         raise NotImplementedError(f"Subclass {self.__class__.__name__} must implement this method.")
 
     @abstractmethod
-    def post_process(self, data: Any) -> Union[Image, Any]:
+    def post_process(self, data: Any, *args, **kwargs) -> Union[Image, Any, Tuple[Any, ...], Dict[Any, Any]]:
         """Transform the prediction results from the model(s).
 
         This method must be overridden by a derived class.