test(General): finished unit test for file handler, fixed unit test for backpressure handler

amqp/adapter/file_handler.py

@@ -158,6 +158,39 @@ class StreamingFileHandler(FileHandler):
     async def _get_connection(self, rabbitmq_url, loop):
         return await connect_robust(rabbitmq_url, loop=loop)
 
+    async def _download_from_queue_to_file(
+        self, rabbitmq_url, queue_name, filepath, loop
+    ) -> tuple[int, int] | None:
+        """
+        Internal method for downloading a file from a queue to disk.
+        Return (file size, eof status).
+        """
+        # Here the normal mock doesn't work since this method
+        # is async and will be run in another loop,
+        # out of the scope of method mock.
+        connection = await self._get_connection(rabbitmq_url, loop=loop)
+        _file_size = 0
+        _eof = IN_PROGRESS
+        try:
+            channel: AbstractRobustChannel = await connection.channel()
+            queue: AbstractQueue = await channel.get_queue(queue_name, ensure=False)
+            logging_debug(f"Awaiting all file chunks being downloaded for {filepath}")
+            async with queue.iterator() as queue_iter:
+                async for message in queue_iter:
+                    async with message.process():
+                        with open(filepath, "ab") as f:
+                            f.write(message.body)
+                        _file_size = _file_size + message.body_size
+                        _eof = int(message.headers.get("eof", END_OF_FILE))
+                        if _eof != IN_PROGRESS:
+                            break
+        except Exception as e:
+            logging_error(f"File download failed: {e}")
+        finally:
+            await connection.close()
+
+        return _file_size, _eof
+
     async def download_file(
         self, loop, rabbitmq_url, file_info, message_data, output_dir
     ) -> (int, int):
@@ -189,42 +222,46 @@ class StreamingFileHandler(FileHandler):
             f"Downloading file: {filename} (size: {size} bytes) from stream: {stream_name}:{queue_name} to {filepath}"
         )
 
-        async def wrap_rabbit_mq_calls() -> tuple[int, int] | None:
-            # Here the normal mock doesn't work since this method
-            # is async and will be run in another loop,
-            # out of the scope of method mock.
-            connection = await self._get_connection(rabbitmq_url, loop=loop)
-            try:
-                _file_size = 0
-                _eof = IN_PROGRESS
-                channel: AbstractRobustChannel = await connection.channel()
-                queue: AbstractQueue = await channel.get_queue(queue_name, ensure=False)
-                logging_debug(f"Awaiting all file chunks being downloaded for {filename}")
-                async with queue.iterator() as queue_iter:
-                    async for message in queue_iter:
-                        async with message.process():
-                            with open(filepath, "ab") as f:
-                                f.write(message.body)
-                            _file_size = _file_size + message.body_size
-                            _eof = int(message.headers.get("eof", END_OF_FILE))
-                            if _eof != IN_PROGRESS:
-                                break
-            except Exception as e:
-                logging_error(f"File download failed: {e}")
-            finally:
-                await connection.close()
-
-            return _file_size, _eof
-
         if queue_name:
             _local_file_size, _local_eof = await await_result(
-                asyncio.run_coroutine_threadsafe(wrap_rabbit_mq_calls(), loop=loop)
+                asyncio.run_coroutine_threadsafe(
+                    self._download_from_queue_to_file(rabbitmq_url, queue_name, filepath, loop),
+                    loop=loop,
+                )
             )
             return _local_file_size, _local_eof
 
         return 0, CANCELLED
 
-    async def download_buffer(self, loop, rabbitmq_url, queue_name) -> (bytearray, int):
+    async def _download_from_queue_to_buffer(
+        self, rabbitmq_url, queue_name, loop
+    ) -> tuple[bytearray, int]:
+        """
+        Internal method for downloading a file from a queue to RAM.
+        Return (bytearray, eof status).
+        """
+        _connection = await self._get_connection(rabbitmq_url, loop=loop)
+        _eof = IN_PROGRESS
+        _res = bytearray()
+        try:
+            channel: AbstractRobustChannel = await _connection.channel()
+            queue: AbstractQueue = await channel.get_queue(queue_name, ensure=False)
+            logging_debug(f"Awaiting all buffer chunks being downloaded for {queue_name}")
+            async with queue.iterator() as queue_iter:
+                async for message in queue_iter:
+                    async with message.process():
+                        _res.extend(message.body)
+                        _eof = message.headers.get("eof", END_OF_FILE)
+                        if _eof != IN_PROGRESS:
+                            break
+            logging_info(f"Finished downloading buffer {queue_name}")
+        except Exception as e:
+            logging_error(f"Downloading buffer: {e}")
+
+        await _connection.close()
+        return _res, _eof
+
+    async def download_buffer(self, loop, rabbitmq_url, queue_name: str | None) -> (bytearray, int):
         """
         Asynchronously downloads a bytearay from RabbitMQ based on the provided file information.
         It assumes entire array is in the first (and only) chunk
@@ -235,31 +272,10 @@ class StreamingFileHandler(FileHandler):
         """
         logging_debug(f"Downloading buffer from stream: {queue_name}")
         if queue_name:
-
-            async def wrap_rabbit_mq_calls() -> tuple[bytearray, int]:
-                _connection = await connect_robust(rabbitmq_url, loop=loop)
-                _eof = IN_PROGRESS
-                _res = bytearray()
-                try:
-                    channel: AbstractRobustChannel = await _connection.channel()
-                    queue: AbstractQueue = await channel.get_queue(queue_name, ensure=False)
-                    logging_debug(f"Awaiting all buffer chunks being downloaded for {queue_name}")
-                    async with queue.iterator() as queue_iter:
-                        async for message in queue_iter:
-                            async with message.process():
-                                _res.extend(message.body)
-                                _eof = message.headers.get("eof", END_OF_FILE)
-                                if _eof != IN_PROGRESS:
-                                    break
-                    logging_info(f"Finished downloading buffer {queue_name}")
-                except Exception as e:
-                    logging_error(f"Downloading buffer: {e}")
-
-                await _connection.close()
-                return _res, _eof
-
             _local_res, _local_eof = await await_result(
-                asyncio.run_coroutine_threadsafe(wrap_rabbit_mq_calls(), loop=loop)
+                asyncio.run_coroutine_threadsafe(
+                    self._download_from_queue_to_buffer(rabbitmq_url, queue_name, loop), loop=loop
+                )
             )
             return _local_res, _local_eof
         return 0, CANCELLED