From 8b231d767c676904edf0ff345d9005a753a4fc68 Mon Sep 17 00:00:00 2001 From: "Zhenyang (Daniel) Feng" Date: Tue, 21 Jan 2025 21:10:32 -0500 Subject: [PATCH] restructure data --- .gitignore | 5 ++ README.md | 8 +- code/sam_utils.py | 2 +- code/segment.py | 8 +- requirements.txt | 1 - segment-anything-2/.clang-format | 85 +++++++++++++++++++ segment-anything-2/.gitignore | 24 ++++++ segment-anything-2/LICENSE | 20 +++++ segment-anything-2/README.md | 3 + segment-anything-2/pyproject.toml | 6 ++ {sam2 => segment-anything-2/sam2}/__init__.py | 0 .../sam2}/automatic_mask_generator.py | 0 .../sam2}/build_sam.py | 0 .../sam2}/csrc/connected_components.cu | 0 .../sam2}/modeling/__init__.py | 0 .../sam2}/modeling/backbones/__init__.py | 0 .../sam2}/modeling/backbones/hieradet.py | 0 .../sam2}/modeling/backbones/image_encoder.py | 0 .../sam2}/modeling/backbones/utils.py | 0 .../sam2}/modeling/memory_attention.py | 0 .../sam2}/modeling/memory_encoder.py | 0 .../sam2}/modeling/position_encoding.py | 0 .../sam2}/modeling/sam/__init__.py | 0 .../sam2}/modeling/sam/mask_decoder.py | 0 .../sam2}/modeling/sam/prompt_encoder.py | 0 .../sam2}/modeling/sam/transformer.py | 0 .../sam2}/modeling/sam2_base.py | 0 .../sam2}/modeling/sam2_utils.py | 0 .../sam2}/sam2_image_predictor.py | 0 .../sam2}/sam2_video_predictor.py | 0 .../sam2}/utils/__init__.py | 0 .../sam2}/utils/amg.py | 0 .../sam2}/utils/misc.py | 0 .../sam2}/utils/transforms.py | 0 .../sam2_configs}/__init__.py | 0 .../sam2_configs}/sam2_hiera_b+.yaml | 0 .../sam2_configs}/sam2_hiera_l.yaml | 0 .../sam2_configs}/sam2_hiera_s.yaml | 0 .../sam2_configs}/sam2_hiera_t.yaml | 0 {sam2 => segment-anything-2}/setup.py | 0 40 files changed, 156 insertions(+), 6 deletions(-) create mode 100644 segment-anything-2/.clang-format create mode 100644 segment-anything-2/.gitignore create mode 100644 segment-anything-2/LICENSE create mode 100644 segment-anything-2/README.md create mode 100644 segment-anything-2/pyproject.toml rename {sam2 => segment-anything-2/sam2}/__init__.py (100%) rename {sam2 => segment-anything-2/sam2}/automatic_mask_generator.py (100%) rename {sam2 => segment-anything-2/sam2}/build_sam.py (100%) rename {sam2 => segment-anything-2/sam2}/csrc/connected_components.cu (100%) rename {sam2 => segment-anything-2/sam2}/modeling/__init__.py (100%) rename {sam2 => segment-anything-2/sam2}/modeling/backbones/__init__.py (100%) rename {sam2 => segment-anything-2/sam2}/modeling/backbones/hieradet.py (100%) rename {sam2 => segment-anything-2/sam2}/modeling/backbones/image_encoder.py (100%) rename {sam2 => segment-anything-2/sam2}/modeling/backbones/utils.py (100%) rename {sam2 => segment-anything-2/sam2}/modeling/memory_attention.py (100%) rename {sam2 => segment-anything-2/sam2}/modeling/memory_encoder.py (100%) rename {sam2 => segment-anything-2/sam2}/modeling/position_encoding.py (100%) rename {sam2 => segment-anything-2/sam2}/modeling/sam/__init__.py (100%) rename {sam2 => segment-anything-2/sam2}/modeling/sam/mask_decoder.py (100%) rename {sam2 => segment-anything-2/sam2}/modeling/sam/prompt_encoder.py (100%) rename {sam2 => segment-anything-2/sam2}/modeling/sam/transformer.py (100%) rename {sam2 => segment-anything-2/sam2}/modeling/sam2_base.py (100%) rename {sam2 => segment-anything-2/sam2}/modeling/sam2_utils.py (100%) rename {sam2 => segment-anything-2/sam2}/sam2_image_predictor.py (100%) rename {sam2 => segment-anything-2/sam2}/sam2_video_predictor.py (100%) rename {sam2 => segment-anything-2/sam2}/utils/__init__.py (100%) rename {sam2 => segment-anything-2/sam2}/utils/amg.py (100%) rename {sam2 => segment-anything-2/sam2}/utils/misc.py (100%) rename {sam2 => segment-anything-2/sam2}/utils/transforms.py (100%) rename {sam2_configs => segment-anything-2/sam2_configs}/__init__.py (100%) rename {sam2_configs => segment-anything-2/sam2_configs}/sam2_hiera_b+.yaml (100%) rename {sam2_configs => segment-anything-2/sam2_configs}/sam2_hiera_l.yaml (100%) rename {sam2_configs => segment-anything-2/sam2_configs}/sam2_hiera_s.yaml (100%) rename {sam2_configs => segment-anything-2/sam2_configs}/sam2_hiera_t.yaml (100%) rename {sam2 => segment-anything-2}/setup.py (100%) diff --git a/.gitignore b/.gitignore index 10b10b9..06166b4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,10 @@ __pycache__/ +data/ +out/ +src/ +GroundingDINO/ *.jpg +*.gif *.png *.pt *.pth diff --git a/README.md b/README.md index e898048..4a21f29 100644 --- a/README.md +++ b/README.md @@ -20,18 +20,22 @@ To use SST, the following setup must be ran on a GPU enabled machine. The code r Example Conda Environment Setup: ```bash +# clone repo +(git clone https://github.com/Imageomics/SST.git && cd SST) # Create conda environment conda create --name sst python=3.10.14 conda activate sst -# Download corresponding torch torchvision version +# Download PyTorch corresponding to the CUDA version of the GPU ... +# Download and setup GroundingDINO +(git clone https://github.com/IDEA-Research/GroundingDINO.git && cd GroundingDINO/ && pip install -e .) # Download required python packages pip install -r requirements.txt --no-dependencies # Download model checkpoints (cd checkpoints && ./download_ckpts.sh) (cd checkpoints && wget -q https://github.com/IDEA-Research/GroundingDINO/releases/download/v0.1.0-alpha/groundingdino_swint_ogc.pth) # Install SAM 2 -(cd sam2 && pip install -e .) +(cd segment-anything-2 && pip install -e .) ``` ## 🧑‍💻 Usage diff --git a/code/sam_utils.py b/code/sam_utils.py index 09edc2c..f531ea8 100644 --- a/code/sam_utils.py +++ b/code/sam_utils.py @@ -147,7 +147,7 @@ def show_anns(anns, color=None, borders=True): ax.imshow(img) -def build_sam2_predictor(checkpoint="../../checkpoints/sam2_hiera_large.pt", model_cfg="../../sam2_configs/sam2_hiera_l.yaml"): +def build_sam2_predictor(checkpoint="checkpoints/sam2_hiera_large.pt", model_cfg="sam2_hiera_l"): device = "cuda" if torch.cuda.is_available() else "cpu" video_predictor = build_sam2_video_predictor(model_cfg, checkpoint, device=device, apply_postprocessing=False) return video_predictor diff --git a/code/segment.py b/code/segment.py index 8383a0d..c257899 100644 --- a/code/segment.py +++ b/code/segment.py @@ -1,4 +1,3 @@ -#%% import os import numpy as np import matplotlib.pyplot as plt @@ -24,6 +23,7 @@ output_format = args.output_format # load the support image and mask +print ("Loading support image and mask...") support_image = cv2.imread(support_image_path)[..., ::-1] support_mask = cv2.imread(support_mask_path, cv2.IMREAD_GRAYSCALE) support_masks = [support_mask == i for i in range(1, support_mask.max()+1)] @@ -36,11 +36,13 @@ video_predictor = sam_utils.build_sam2_predictor() # load the support image and mask +print ("Inferring the masks...") state = sam_utils.load_masks(video_predictor, query_images, support_image, support_masks, verbose=True) frames_info = sam_utils.propagate_masks(video_predictor, state, verbose=True) # visualize the results output_imgs = [] +print ("Visualizing the results...") for i, frame in enumerate(frames_info): plt.clf() plt.figure(figsize=(10, 10)) @@ -61,8 +63,10 @@ if not os.path.exists(output_folder): os.makedirs(output_folder) if output_format == 'gif': - output_imgs[0].save(output_folder, save_all=True, append_images=output_imgs[1:], loop=0, duration=1000) + output_imgs[0].save(os.path.join(output_folder, "out.gif"), save_all=True, append_images=output_imgs[1:], loop=0, duration=1000) else: for i, img in enumerate(output_imgs): img.save(os.path.join(output_folder, f"{i:06d}.png")) +print ("Done! The output is saved in", output_folder) + diff --git a/requirements.txt b/requirements.txt index 2a7e6c4..3ffc2f0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -69,7 +69,6 @@ GitPython==3.1.43 google-pasta==0.2.0 gradio==3.32.0 gradio_client==0.2.5 --e git+https://github.com/IDEA-Research/GroundingDINO.git@856dde20aee659246248e20734ef9ba5214f5e44#egg=groundingdino grpcio==1.66.1 h11==0.14.0 h5py==3.11.0 diff --git a/segment-anything-2/.clang-format b/segment-anything-2/.clang-format new file mode 100644 index 0000000..39b1b3d --- /dev/null +++ b/segment-anything-2/.clang-format @@ -0,0 +1,85 @@ +AccessModifierOffset: -1 +AlignAfterOpenBracket: AlwaysBreak +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlinesLeft: true +AlignOperands: false +AlignTrailingComments: false +AllowAllParametersOfDeclarationOnNextLine: false +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: Empty +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: true +BinPackArguments: false +BinPackParameters: false +BraceWrapping: + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + BeforeCatch: false + BeforeElse: false + IndentBraces: false +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Attach +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: false +ColumnLimit: 80 +CommentPragmas: '^ IWYU pragma:' +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DerivePointerAlignment: false +DisableFormat: false +ForEachMacros: [ FOR_EACH, FOR_EACH_R, FOR_EACH_RANGE, ] +IncludeCategories: + - Regex: '^<.*\.h(pp)?>' + Priority: 1 + - Regex: '^<.*' + Priority: 2 + - Regex: '.*' + Priority: 3 +IndentCaseLabels: true +IndentWidth: 2 +IndentWrappedFunctionNames: false +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: false +PenaltyBreakBeforeFirstCallParameter: 1 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 200 +PointerAlignment: Left +ReflowComments: true +SortIncludes: true +SpaceAfterCStyleCast: false +SpaceBeforeAssignmentOperators: true +SpaceBeforeParens: ControlStatements +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 1 +SpacesInAngles: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: Cpp11 +TabWidth: 8 +UseTab: Never diff --git a/segment-anything-2/.gitignore b/segment-anything-2/.gitignore new file mode 100644 index 0000000..0eba7cd --- /dev/null +++ b/segment-anything-2/.gitignore @@ -0,0 +1,24 @@ +.vscode/ +.DS_Store +__pycache__/ +*-checkpoint.ipynb +.venv +*.egg* +build/* +_C.* +outputs/* +*.pt +*.sh +data/* +*.jpg +*.png +results/* +sav_dataset/* +*.gz +*.zip +*.out +*.log +*.pkl +*.pdf +*.gif + diff --git a/segment-anything-2/LICENSE b/segment-anything-2/LICENSE new file mode 100644 index 0000000..e840fff --- /dev/null +++ b/segment-anything-2/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2023-2024 Imageomics Institute + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/segment-anything-2/README.md b/segment-anything-2/README.md new file mode 100644 index 0000000..0ad9d6b --- /dev/null +++ b/segment-anything-2/README.md @@ -0,0 +1,3 @@ +# SST + +Repository for the paper Static Segmentation by Tracking: A Frustratingly Label-Efficient Approach to Fine-Grained Segmentation. diff --git a/segment-anything-2/pyproject.toml b/segment-anything-2/pyproject.toml new file mode 100644 index 0000000..f7e8652 --- /dev/null +++ b/segment-anything-2/pyproject.toml @@ -0,0 +1,6 @@ +[build-system] +requires = [ + "setuptools>=61.0", + "torch>=2.3.1", + ] +build-backend = "setuptools.build_meta" diff --git a/sam2/__init__.py b/segment-anything-2/sam2/__init__.py similarity index 100% rename from sam2/__init__.py rename to segment-anything-2/sam2/__init__.py diff --git a/sam2/automatic_mask_generator.py b/segment-anything-2/sam2/automatic_mask_generator.py similarity index 100% rename from sam2/automatic_mask_generator.py rename to segment-anything-2/sam2/automatic_mask_generator.py diff --git a/sam2/build_sam.py b/segment-anything-2/sam2/build_sam.py similarity index 100% rename from sam2/build_sam.py rename to segment-anything-2/sam2/build_sam.py diff --git a/sam2/csrc/connected_components.cu b/segment-anything-2/sam2/csrc/connected_components.cu similarity index 100% rename from sam2/csrc/connected_components.cu rename to segment-anything-2/sam2/csrc/connected_components.cu diff --git a/sam2/modeling/__init__.py b/segment-anything-2/sam2/modeling/__init__.py similarity index 100% rename from sam2/modeling/__init__.py rename to segment-anything-2/sam2/modeling/__init__.py diff --git a/sam2/modeling/backbones/__init__.py b/segment-anything-2/sam2/modeling/backbones/__init__.py similarity index 100% rename from sam2/modeling/backbones/__init__.py rename to segment-anything-2/sam2/modeling/backbones/__init__.py diff --git a/sam2/modeling/backbones/hieradet.py b/segment-anything-2/sam2/modeling/backbones/hieradet.py similarity index 100% rename from sam2/modeling/backbones/hieradet.py rename to segment-anything-2/sam2/modeling/backbones/hieradet.py diff --git a/sam2/modeling/backbones/image_encoder.py b/segment-anything-2/sam2/modeling/backbones/image_encoder.py similarity index 100% rename from sam2/modeling/backbones/image_encoder.py rename to segment-anything-2/sam2/modeling/backbones/image_encoder.py diff --git a/sam2/modeling/backbones/utils.py b/segment-anything-2/sam2/modeling/backbones/utils.py similarity index 100% rename from sam2/modeling/backbones/utils.py rename to segment-anything-2/sam2/modeling/backbones/utils.py diff --git a/sam2/modeling/memory_attention.py b/segment-anything-2/sam2/modeling/memory_attention.py similarity index 100% rename from sam2/modeling/memory_attention.py rename to segment-anything-2/sam2/modeling/memory_attention.py diff --git a/sam2/modeling/memory_encoder.py b/segment-anything-2/sam2/modeling/memory_encoder.py similarity index 100% rename from sam2/modeling/memory_encoder.py rename to segment-anything-2/sam2/modeling/memory_encoder.py diff --git a/sam2/modeling/position_encoding.py b/segment-anything-2/sam2/modeling/position_encoding.py similarity index 100% rename from sam2/modeling/position_encoding.py rename to segment-anything-2/sam2/modeling/position_encoding.py diff --git a/sam2/modeling/sam/__init__.py b/segment-anything-2/sam2/modeling/sam/__init__.py similarity index 100% rename from sam2/modeling/sam/__init__.py rename to segment-anything-2/sam2/modeling/sam/__init__.py diff --git a/sam2/modeling/sam/mask_decoder.py b/segment-anything-2/sam2/modeling/sam/mask_decoder.py similarity index 100% rename from sam2/modeling/sam/mask_decoder.py rename to segment-anything-2/sam2/modeling/sam/mask_decoder.py diff --git a/sam2/modeling/sam/prompt_encoder.py b/segment-anything-2/sam2/modeling/sam/prompt_encoder.py similarity index 100% rename from sam2/modeling/sam/prompt_encoder.py rename to segment-anything-2/sam2/modeling/sam/prompt_encoder.py diff --git a/sam2/modeling/sam/transformer.py b/segment-anything-2/sam2/modeling/sam/transformer.py similarity index 100% rename from sam2/modeling/sam/transformer.py rename to segment-anything-2/sam2/modeling/sam/transformer.py diff --git a/sam2/modeling/sam2_base.py b/segment-anything-2/sam2/modeling/sam2_base.py similarity index 100% rename from sam2/modeling/sam2_base.py rename to segment-anything-2/sam2/modeling/sam2_base.py diff --git a/sam2/modeling/sam2_utils.py b/segment-anything-2/sam2/modeling/sam2_utils.py similarity index 100% rename from sam2/modeling/sam2_utils.py rename to segment-anything-2/sam2/modeling/sam2_utils.py diff --git a/sam2/sam2_image_predictor.py b/segment-anything-2/sam2/sam2_image_predictor.py similarity index 100% rename from sam2/sam2_image_predictor.py rename to segment-anything-2/sam2/sam2_image_predictor.py diff --git a/sam2/sam2_video_predictor.py b/segment-anything-2/sam2/sam2_video_predictor.py similarity index 100% rename from sam2/sam2_video_predictor.py rename to segment-anything-2/sam2/sam2_video_predictor.py diff --git a/sam2/utils/__init__.py b/segment-anything-2/sam2/utils/__init__.py similarity index 100% rename from sam2/utils/__init__.py rename to segment-anything-2/sam2/utils/__init__.py diff --git a/sam2/utils/amg.py b/segment-anything-2/sam2/utils/amg.py similarity index 100% rename from sam2/utils/amg.py rename to segment-anything-2/sam2/utils/amg.py diff --git a/sam2/utils/misc.py b/segment-anything-2/sam2/utils/misc.py similarity index 100% rename from sam2/utils/misc.py rename to segment-anything-2/sam2/utils/misc.py diff --git a/sam2/utils/transforms.py b/segment-anything-2/sam2/utils/transforms.py similarity index 100% rename from sam2/utils/transforms.py rename to segment-anything-2/sam2/utils/transforms.py diff --git a/sam2_configs/__init__.py b/segment-anything-2/sam2_configs/__init__.py similarity index 100% rename from sam2_configs/__init__.py rename to segment-anything-2/sam2_configs/__init__.py diff --git a/sam2_configs/sam2_hiera_b+.yaml b/segment-anything-2/sam2_configs/sam2_hiera_b+.yaml similarity index 100% rename from sam2_configs/sam2_hiera_b+.yaml rename to segment-anything-2/sam2_configs/sam2_hiera_b+.yaml diff --git a/sam2_configs/sam2_hiera_l.yaml b/segment-anything-2/sam2_configs/sam2_hiera_l.yaml similarity index 100% rename from sam2_configs/sam2_hiera_l.yaml rename to segment-anything-2/sam2_configs/sam2_hiera_l.yaml diff --git a/sam2_configs/sam2_hiera_s.yaml b/segment-anything-2/sam2_configs/sam2_hiera_s.yaml similarity index 100% rename from sam2_configs/sam2_hiera_s.yaml rename to segment-anything-2/sam2_configs/sam2_hiera_s.yaml diff --git a/sam2_configs/sam2_hiera_t.yaml b/segment-anything-2/sam2_configs/sam2_hiera_t.yaml similarity index 100% rename from sam2_configs/sam2_hiera_t.yaml rename to segment-anything-2/sam2_configs/sam2_hiera_t.yaml diff --git a/sam2/setup.py b/segment-anything-2/setup.py similarity index 100% rename from sam2/setup.py rename to segment-anything-2/setup.py