Merge pull request #763 from superannotateai/FRIDAY_3601

Add ability to donwload annotations for multimodal projects

Author: VaghinakDev

src/superannotate/lib/app/interface/sdk_interface.py

@@ -3809,6 +3809,7 @@ def download_annotations(
         items: Optional[List[NotEmptyStr]] = None,
         recursive: bool = False,
         callback: Callable = None,
+        data_spec: Literal["default", "multimodal"] = "default",
     ):
         """Downloads annotation JSON files of the selected items to the local directory.
 
@@ -3834,6 +3835,31 @@ def download_annotations(
          The function receives each annotation as an argument and the returned value will be applied to the download.
         :type callback: callable
 
+        :param data_spec: Specifies the format for processing and transforming annotations before upload.
+
+            Options are:
+                    - default: Retains the annotations in their original format.
+                    - multimodal: Converts annotations for multimodal projects, optimizing for
+                                     compact and multimodal-specific data representation.
+
+        :type data_spec: str, optional
+
+        Example Usage of Multimodal Projects::
+
+            from superannotate import SAClient
+
+
+            sa = SAClient()
+
+            # Call the get_annotations function
+            response = sa.download_annotations(
+                project="project1/folder1",
+                path="path/to/download",
+                items=["item_1", "item_2"],
+                data_spec='multimodal'
+            )
+
+
         :return: local path of the downloaded annotations folder.
         :rtype: str
         """
@@ -3846,6 +3872,7 @@ def download_annotations(
             recursive=recursive,
             item_names=items,
             callback=callback,
+            transform_version="llmJsonV2" if data_spec == "multimodal" else None,
         )
         if response.errors:
             raise AppException(response.errors)

src/superannotate/lib/core/serviceproviders.py

@@ -501,6 +501,7 @@ async def download_big_annotation(
         download_path: str,
         item: entities.BaseItemEntity,
         callback: Callable = None,
+        transform_version: str = None,
     ):
         raise NotImplementedError
 
@@ -513,6 +514,7 @@ async def download_small_annotations(
         download_path: str,
         item_ids: List[int],
         callback: Callable = None,
+        transform_version: str = None,
     ):
         raise NotImplementedError
 

src/superannotate/lib/core/usecases/annotations.py

@@ -1634,6 +1634,7 @@ def __init__(
         item_names: List[str],
         service_provider: BaseServiceProvider,
         callback: Callable = None,
+        transform_version=None,
     ):
         super().__init__(reporter)
         self._config = config
@@ -1645,6 +1646,7 @@ def __init__(
         self._service_provider = service_provider
         self._callback = callback
         self._big_file_queue = None
+        self._transform_version = transform_version
 
     def validate_items(self):
         if self._item_names:
@@ -1724,6 +1726,7 @@ async def download_small_annotations(
             reporter=self.reporter,
             download_path=f"{export_path}{'/' + self._folder.name if not self._folder.is_root else ''}",
             callback=self._callback,
+            transform_version=self._transform_version,
         )
 
     async def run_workers(

src/superannotate/lib/infrastructure/controller.py

@@ -932,6 +932,7 @@ def download(
         recursive: bool,
         item_names: Optional[List[str]],
         callback: Optional[Callable],
+        transform_version: str,
     ):
         use_case = usecases.DownloadAnnotations(
             config=self._config,
@@ -943,6 +944,7 @@ def download(
             item_names=item_names,
             service_provider=self.service_provider,
             callback=callback,
+            transform_version=transform_version,
         )
         return use_case.execute()
 

src/superannotate/lib/infrastructure/services/annotation.py

@@ -67,7 +67,9 @@ def get_schema(self, project_type: int, version: str):
             },
         )
 
-    async def _sync_large_annotation(self, team_id, project_id, item_id):
+    async def _sync_large_annotation(
+        self, team_id, project_id, item_id, transform_version: str = None
+    ):
         sync_params = {
             "team_id": team_id,
             "project_id": project_id,
@@ -77,6 +79,8 @@ async def _sync_large_annotation(self, team_id, project_id, item_id):
             "current_source": "main",
             "desired_source": "secondary",
         }
+        if transform_version:
+            sync_params["transform_version"] = transform_version
         sync_url = urljoin(
             self.get_assets_provider_url(),
             self.URL_START_FILE_SYNC.format(item_id=item_id),
@@ -120,11 +124,12 @@ async def get_big_annotation(
             "annotation_type": "MAIN",
             "version": "V1.00",
         }
-        if transform_version:
-            query_params["desired_transform_version"] = transform_version
 
         await self._sync_large_annotation(
-            team_id=project.team_id, project_id=project.id, item_id=item.id
+            team_id=project.team_id,
+            project_id=project.id,
+            item_id=item.id,
+            transform_version=transform_version,
         )
 
         async with AIOHttpSession(
@@ -202,6 +207,7 @@ async def download_big_annotation(
         download_path: str,
         item: entities.BaseItemEntity,
         callback: Callable = None,
+        transform_version: str = None,
     ):
         item_id = item.id
         item_name = item.name
@@ -218,7 +224,10 @@ async def download_big_annotation(
         )
 
         await self._sync_large_annotation(
-            team_id=project.team_id, project_id=project.id, item_id=item_id
+            team_id=project.team_id,
+            project_id=project.id,
+            item_id=item_id,
+            transform_version=transform_version,
         )
 
         async with AIOHttpSession(
@@ -252,12 +261,15 @@ async def download_small_annotations(
         download_path: str,
         item_ids: List[int],
         callback: Callable = None,
+        transform_version: str = None,
     ):
         query_params = {
             "team_id": project.team_id,
             "project_id": project.id,
             "folder_id": folder.id,
         }
+        if transform_version:
+            query_params["transform_version"] = transform_version
         handler = StreamedAnnotations(
             headers=self.client.default_headers,
             reporter=reporter,

tests/data_set/editor_templates/from1_classes.json renamed to tests/data_set/editor_templates/form1_classes.json

@@ -1,5 +1,6 @@
 import json
 import os
+import tempfile
 import time
 from pathlib import Path
 
@@ -133,23 +134,19 @@ def test_upload_large_annotations(self):
         ) == 5
 
 
-class MultiModalUploadAnnotations(BaseTestCase):
+class MultiModalUploadDownloadAnnotations(BaseTestCase):
     PROJECT_NAME = "TestMultimodalUploadAnnotations"
     PROJECT_TYPE = "Multimodal"
     PROJECT_DESCRIPTION = "DESCRIPTION"
-    EDITOR_TEMPLATE_PATH = os.path.join(
-        Path(__file__).parent.parent.parent, "data_set/editor_templates/form1.json"
-    )
-    JSONL_ANNOTATIONS_PATH = os.path.join(
-        DATA_SET_PATH, "multimodal/annotations/jsonl/form1.jsonl"
-    )
-    JSONL_ANNOTATIONS_WITH_CATEGORIES_PATH = os.path.join(
-        DATA_SET_PATH, "multimodal/annotations/jsonl/form1_with_categories.jsonl"
-    )
-    CLASSES_TEMPLATE_PATH = os.path.join(
-        Path(__file__).parent.parent.parent,
-        "data_set/editor_templates/from1_classes.json",
+    BASE_PATH = Path(__file__).parent.parent.parent
+    DATA_SET_PATH = BASE_PATH / "data_set"
+
+    EDITOR_TEMPLATE_PATH = DATA_SET_PATH / "editor_templates/form1.json"
+    JSONL_ANNOTATIONS_PATH = DATA_SET_PATH / "multimodal/annotations/jsonl/form1.jsonl"
+    JSONL_ANNOTATIONS_WITH_CATEGORIES_PATH = (
+        DATA_SET_PATH / "multimodal/annotations/jsonl/form1_with_categories.jsonl"
     )
+    CLASSES_TEMPLATE_PATH = DATA_SET_PATH / "editor_templates" / "form1_classes.json"
 
     def setUp(self, *args, **kwargs):
         self.tearDown()
@@ -244,3 +241,32 @@ def test_upload_with_integer_names(self):
                 sa.get_annotations(
                     f"{self.PROJECT_NAME}/test_folder", data_spec="multimodal"
                 )
+
+    def test_download_annotations(self):
+        with open(self.JSONL_ANNOTATIONS_PATH) as f:
+            data = [json.loads(line) for line in f]
+            sa.upload_annotations(
+                self.PROJECT_NAME, annotations=data, data_spec="multimodal"
+            )
+
+        annotations = sa.get_annotations(
+            f"{self.PROJECT_NAME}/test_folder", data_spec="multimodal"
+        )
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            sa.download_annotations(
+                f"{self.PROJECT_NAME}/test_folder", path=tmpdir, data_spec="multimodal"
+            )
+            downloaded_files = list(Path(f"{tmpdir}/test_folder").glob("*.json"))
+            assert len(downloaded_files) > 0, "No annotations were downloaded"
+            downloaded_data = []
+            for file_path in downloaded_files:
+                with open(file_path) as f:
+                    downloaded_data.append(json.load(f))
+
+            assert len(downloaded_data) == len(
+                annotations
+            ), "Mismatch in annotation count"
+            assert (
+                downloaded_data == annotations
+            ), "Downloaded annotations do not match uploaded annotations"

tests/integration/annotations/test_upload_annotations.py

@@ -19,7 +19,7 @@ class TestMultimodalProjectBasic(BaseTestCase):
     )
     CLASSES_TEMPLATE_PATH = os.path.join(
         Path(__file__).parent.parent.parent,
-        "data_set/editor_templates/from1_classes.json",
+        "data_set/editor_templates/form1_classes.json",
     )
 
     def setUp(self, *args, **kwargs):