feat: add compound file ordinal endpoints update (#56)

Author: PJEstrada

sdk/diffgram/file/compound_file.py

@@ -6,6 +6,7 @@
 
 
 class CompoundChildFile:
+    id: int
     path: str
     url: str
     blob_path: str
@@ -24,8 +25,10 @@ class CompoundChildFile:
     video_split_duration: int
     local_id: str
     ordinal: int
+    root_file: 'CompoundFile'
 
     def __init__(self,
+                 root_file: 'CompoundFile',
                  child_file_type: str,
                  path: str = None,
                  url: str = None,
@@ -42,7 +45,9 @@ def __init__(self,
                  assume_new_instances_machine_made: bool = None,
                  convert_names_to_label_files: bool = None,
                  video_split_duration: int = None,
-                 ordinal: int = 0):
+                 ordinal: int = 0,
+                 id: int = None):
+        self.root_file = root_file
         self.child_file_type = child_file_type
         self.path = path
         self.url = url
@@ -61,6 +66,38 @@ def __init__(self,
         self.video_split_duration = video_split_duration
         self.local_id = str(uuid4())
         self.ordinal = ordinal
+        self.id = None
+
+    def __str__(self):
+        return f'<CompoundChildFile id={self.id} ordinal={self.ordinal} file_name={self.file_name} child_file_type={self.child_file_type}>'
+    def refresh_from_data_dict(self, data: dict):
+        if not data:
+            return
+        for key in data:
+            setattr(self, key, data[key])
+
+    def update(self):
+        """
+            Syncs child file data with backend API
+        :return:
+        """
+        payload = {
+            'ordinal': self.ordinal,
+        }
+        client = self.root_file.project
+        endpoint = f"/api/v1/project/{self.root_file.project.project_string_id}/file/{self.id}/update-metadata"
+
+        response = client.session.put(
+            client.host + endpoint,
+            json = payload)
+
+        client.handle_errors(response)
+
+        data = response.json()
+        new_file_data = data['file']
+
+        self.refresh_from_data_dict(data = new_file_data)
+        return self
 
     def set_ordinal(self, value: int):
         self.ordinal = value
@@ -69,23 +106,57 @@ def set_ordinal(self, value: int):
 class CompoundFile:
     project: Project
     parent_file_data: dict
-    child_files_to_upload: List[CompoundChildFile]
+    child_files: List[CompoundChildFile]
 
     def __init__(self, project: Project, name: str, directory_id: int):
         self.project = project
         self.name = name
         self.directory_id = directory_id
-        self.child_files_to_upload = []
+        self.child_files = []
+
+    @staticmethod
+    def from_dict(project: Project, dir_id: int, dict_data: dict):
+        result = CompoundFile(project = project, name = dict_data.get('original_filename'), directory_id = dir_id)
+        result.__refresh_compound_file_from_data_dict(data = dict_data)
+        child_files = result.__fetch_child_files()
+        result.child_files = child_files
+        return result
+
+    def __fetch_child_files(self) -> List[CompoundChildFile]:
+        client = self.project
+        endpoint = f"/api/v1/project/{self.project.project_string_id}/file/{self.id}/child-files"
+
+        response = client.session.get(client.host + endpoint)
+
+        client.handle_errors(response)
+
+        data = response.json()
+        child_files_data = data['child_files']
+        result = []
+        for elm in child_files_data:
+            child_file = CompoundChildFile(root_file = self, child_file_type = elm.get('type'))
+            child_file.refresh_from_data_dict(data = elm)
+            result.append(child_file)
+        return result
+
+    def update_all(self) -> bool:
+        """
+            Syncs parent and child metadata with backend API.
+        :return: True/False
+        """
+        for child in self.child_files:
+            child.update()
+        return True
 
     def __refresh_compound_file_from_data_dict(self, data: dict):
         if not data:
             return
         for key in data:
             setattr(self, key, data[key])
 
-    def remove_compound_file(self, child_file: CompoundChildFile) -> List[CompoundChildFile]:
-        self.child_files_to_upload.remove(child_file)
-        return self.child_files_to_upload
+    def remove_child_file(self, child_file: CompoundChildFile) -> List[CompoundChildFile]:
+        self.child_files.remove(child_file)
+        return self.child_files
 
     def __create_compound_parent_file(self):
         url = f'/api/v1/project/{self.project.project_string_id}/file/new-compound'
@@ -110,7 +181,8 @@ def __create_child_file(self, child_file: CompoundChildFile):
                 frame_packet_map = child_file.frame_packet_map,
                 assume_new_instances_machine_made = child_file.assume_new_instances_machine_made,
                 convert_names_to_label_files = child_file.convert_names_to_label_files,
-                parent_file_id = self.parent_file_data.get('id')
+                parent_file_id = self.parent_file_data.get('id'),
+                ordinal = child_file.ordinal,
             )
         elif child_file.child_file_type == 'from_url':
             return self.project.file.from_url(
@@ -121,7 +193,8 @@ def __create_child_file(self, child_file: CompoundChildFile):
                 video_split_duration = child_file.video_split_duration,
                 instance_list = child_file.instance_list,
                 frame_packet_map = child_file.frame_packet_map,
-                parent_file_id = self.parent_file_data.get('id')
+                parent_file_id = self.parent_file_data.get('id'),
+                ordinal = child_file.ordinal,
             )
         elif child_file.child_file_type == 'from_blob_path':
             return self.project.file.from_blob_path(
@@ -132,7 +205,8 @@ def __create_child_file(self, child_file: CompoundChildFile):
                 instance_list = child_file.instance_list,
                 file_name = child_file.file_name,
                 frame_packet_map = child_file.frame_packet_map,
-                parent_file_id = self.parent_file_data.get('id')
+                parent_file_id = self.parent_file_data.get('id'),
+                ordinal = child_file.ordinal,
             )
 
     def add_child_from_local(self,
@@ -141,17 +215,21 @@ def add_child_from_local(self,
                              frame_packet_map: dict = None,
                              assume_new_instances_machine_made: bool = True,
                              convert_names_to_label_files: bool = True,
-                             ordinal: int = 0):
+                             ordinal: int = None):
+        if ordinal is None:
+            ordinal = len(self.child_files)
         new_child_file = CompoundChildFile(
+            root_file = self,
             child_file_type = "from_local",
             path = path,
             directory_id = self.directory_id,
             instance_list = instance_list,
             frame_packet_map = frame_packet_map,
             assume_new_instances_machine_made = assume_new_instances_machine_made,
-            convert_names_to_label_files = convert_names_to_label_files
+            convert_names_to_label_files = convert_names_to_label_files,
+            ordinal = ordinal
         )
-        self.child_files_to_upload.append(new_child_file)
+        self.child_files.append(new_child_file)
         return new_child_file
 
     def add_child_file_from_url(self,
@@ -162,8 +240,11 @@ def add_child_file_from_url(self,
                                 video_split_duration: int = None,
                                 instance_list: list = None,
                                 frame_packet_map: dict = None,
-                                ordinal: int = 0):
+                                ordinal: int = None):
+        if ordinal is None:
+            ordinal = len(self.child_files)
         new_child_file = CompoundChildFile(
+            root_file = self,
             child_file_type = "from_url",
             url = url,
             media_type = media_type,
@@ -173,8 +254,9 @@ def add_child_file_from_url(self,
             video_split_duration = video_split_duration,
             instance_list = instance_list,
             frame_packet_map = frame_packet_map,
+            ordinal = ordinal
         )
-        self.child_files_to_upload.append(new_child_file)
+        self.child_files.append(new_child_file)
         return new_child_file
 
     def add_child_from_blob_path(self,
@@ -185,9 +267,12 @@ def add_child_from_blob_path(self,
                                  instance_list: list = None,
                                  file_name: str = None,
                                  frame_packet_map: dict = None,
-                                 ordinal: int = 0
+                                 ordinal: int = None
                                  ):
+        if ordinal is None:
+            ordinal = len(self.child_files)
         new_child_file = CompoundChildFile(
+            root_file = self,
             child_file_type = "from_blob_path",
             blob_path = blob_path,
             bucket_name = bucket_name,
@@ -197,15 +282,16 @@ def add_child_from_blob_path(self,
             instance_list = instance_list,
             file_name = file_name,
             frame_packet_map = frame_packet_map,
+            ordinal = ordinal
         )
-        self.child_files_to_upload.append(new_child_file)
+        self.child_files.append(new_child_file)
         return new_child_file
 
     def upload(self):
-        if len(self.child_files_to_upload) == 0:
+        if len(self.child_files) == 0:
             raise AssertionError('Need to add at least one child file to the compound file before calling upload()')
         parent_file_data: dict = self.__create_compound_parent_file()
 
-        for child_file in self.child_files_to_upload:
+        for child_file in self.child_files:
             self.__create_child_file(child_file)
         return parent_file_data

sdk/diffgram/file/file_constructor.py

@@ -44,7 +44,8 @@ def from_local(
             frame_packet_map: dict = None,
             assume_new_instances_machine_made: bool = True,
             convert_names_to_label_files: bool = True,
-            parent_file_id: int = None
+            parent_file_id: int = None,
+            ordinal: int = 0
     ):
         """
         Create a Project file from local path
@@ -56,7 +57,9 @@ def from_local(
 
         files = {'file': (os.path.basename(path), open(path, 'rb'), 'application/octet-stream')}
 
-        json_payload = {}
+        json_payload = {
+            'ordinal': ordinal
+        }
 
         if directory_id is None:
             directory_id = self.client.directory_id
@@ -107,13 +110,15 @@ def __build_packet_payload(self,
                                bucket_name: str = None,
                                file_name: str = None,
                                blob_path: str = None,
-                               parent_file_id: int = None):
+                               parent_file_id: int = None,
+                               ordinal: int = 0):
         packet = {'media': {}}
         packet['media']['url'] = url
         packet['media']['type'] = media_type
 
         # Existing Instances
         packet['frame_packet_map'] = frame_packet_map
+        packet['ordinal'] = ordinal
         packet['type'] = type
         packet['connection_id'] = connection_id
         packet['parent_file_id'] = parent_file_id
@@ -140,7 +145,8 @@ def from_blob_path(self,
                        instance_list: list = None,
                        file_name: str = None,
                        frame_packet_map: dict = None,
-                       parent_file_id: int = None):
+                       parent_file_id: int = None,
+                       ordinal: int = 0):
         """
           Bind a blob path in the given connection ID into Diffgram
         :param blob_path:
@@ -167,7 +173,8 @@ def from_blob_path(self,
             file_name = name,
             directory_id = directory_id,
             type = "from_blob_path",
-            parent_file_id=parent_file_id
+            parent_file_id=parent_file_id,
+            ordinal = ordinal
         )
         self.from_packet(packet = packet)
         return True
@@ -181,7 +188,8 @@ def from_url(
             video_split_duration: int = None,
             instance_list: list = None,  # for Images
             frame_packet_map: dict = None,  # for Video
-            parent_file_id: int = None
+            parent_file_id: int = None,
+            ordinal: int = 0
     ):
         """
 
@@ -214,7 +222,8 @@ def from_url(
             video_split_duration = video_split_duration,
             instance_list = instance_list,
             frame_packet_map = frame_packet_map,
-            parent_file_id = parent_file_id
+            parent_file_id = parent_file_id,
+            ordinal = ordinal
         )
         self.from_packet(packet = packet)
 
@@ -554,7 +563,15 @@ def get_by_id(self,
 
         response_json = response.json()
         file_data = response_json.get(file_response_key)
+        if file_data.get('type') == 'compound':
+            from diffgram.file.compound_file import CompoundFile
+            return CompoundFile.from_dict(
+                project = self.client,
+                dir_id = self.client.directory_id,
+                dict_data = file_data
+            )
 
-        return File.new(
-            client = self.client,
-            file_json = file_data)
+        else:
+            return File.new(
+                client = self.client,
+                file_json = file_data)