Fix pipeline (#10)

This MR fixed the pipeline so it can run on forgejo runner. Currently we have two pipelines set up:

+ One pipeline will run pytest to test the code and collect coverage data, this pipeline will run on all branches and PRs.
+ One pipeline will build the wheel file and publish it to the company's pypl repo, this pipeline will only run on the master branch and develop branch.

This RP also made the following changes to make the pipeline working:

+ Remove `setup.py` and `requirements.txt`, now the project is fully embraced `pyproject.toml`, which according to ChatGPT, is the most modern way of managing a python project.
+ Move all test files to `tests` folder, so it won't be packed into the final wheel file and affect the coverage rate.
+ Change version to `X.Y.Z-dev`, before building the wheel file, the pipeline will automatically add a UTC timestamp after it, so it becomes `X.Y.Z-devYYYYMMDDHHmmSS`. This is added due to the limitation of the pypl repo, unlike maven snapshot versions, you can't push the same version twice with our pypl repository.

Reviewed-on: #10
Reviewed-by: Stanislav Hejny <stanislav.hejny@cleverthis.com>
Co-authored-by: Rui Hu <rui.hu@cleverthis.com>
Co-committed-by: Rui Hu <rui.hu@cleverthis.com>

tests/adapter/test_amq_route_factory.py

@@ -0,0 +1,30 @@
+from unittest import TestCase
+
+from amqp.adapter.amq_route_factory import AMQRouteFactory, NULL_ROUTE
+from amqp.model.model import AMQRoute
+
+
+class TestAMQRouteFactory(TestCase):
+    def test_from_string(self):
+        _f = AMQRouteFactory()
+        _r: AMQRoute = _f.from_string(
+            "amq-clever-pybook::amq-clever-pybook:clever.pybook.localhost.#:5:0"
+        )
+        self.assertEqual("", _r.exchange)
+        self.assertEqual("amq-clever-pybook", _r.queue)
+        self.assertEqual(5, _r.timeout)
+        _r2: AMQRoute = _f.from_string(
+            "amq-clever-pybook:amq-clever-pybook:clever.pybook.localhost.#:5"
+        )
+        self.assertEqual(NULL_ROUTE, _r2)
+
+    def test_validate(self):
+        _f = AMQRouteFactory()
+        self.assertTrue(
+            _f.validate(
+                "amq-clever-pybook::amq-clever-pybook:clever.pybook.localhost.#:5:0"
+            )
+        )
+        self.assertFalse(
+            _f.validate("amq-clever-pybook:amq-clever-pybook:clever.pybook.localhost.#")
+        )

tests/adapter/test_cleverthis_service_adapter.py

@@ -0,0 +1,227 @@
+import unittest
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from aiohttp import ClientResponse
+
+from amqp.adapter.cleverthis_service_adapter import CleverThisServiceAdapter, AMQMessage
+from amqp.config.amq_configuration import AMQConfiguration
+from amqp.model.model import DataMessage, DataResponse
+
+
+class TestCleverThisServiceAdapter(unittest.TestCase):
+    def setUp(self):
+        # Mock AMQConfiguration
+        self.mock_amq_config = AMQConfiguration("./application.properties.local")
+
+        # Mock ClientSession
+        self.mock_session = AsyncMock()
+
+        # Create adapter instance
+        self.adapter = CleverThisServiceAdapter(
+            amq_configuration=self.mock_amq_config, session=self.mock_session
+        )
+
+    def test_init(self):
+        self.assertEqual(self.adapter.service_host, "localhost")
+        self.assertEqual(self.adapter.service_port, 8080)
+        self.assertFalse(self.adapter.require_authenticated_user)
+
+    def test_get_content_type(self):
+        test_cases = [
+            ({"Content-Type": ["application/json"]}, "application/json"),
+            ({"content-type": ["text/xml"]}, "text/xml"),
+            ({"CONTENT-TYPE": ["text/plain; charset=utf-8"]}, "text/plain"),
+            ({}, "application/json"),
+            ({"Other-Header": ["value"]}, "application/json"),
+        ]
+
+        for headers, expected in test_cases:
+            with self.subTest(headers=headers):
+                result = self.adapter.get_content_type(headers)
+                self.assertEqual(result, expected)
+
+    async def test_get_current_user(self):
+        # Base implementation should return None
+        result = await self.adapter.get_current_user("test_token")
+        self.assertIsNone(result)
+
+    @patch.object(CleverThisServiceAdapter, "get_current_user")
+    async def test_on_message_unauthenticated(self, mock_get_user):
+        mock_message = MagicMock(spec=AMQMessage)
+        mock_message.headers = {}
+        mock_message.path = "/some/path"
+        mock_message.method = "GET"
+        mock_message.id = "123"
+        mock_message.trace_info = {}
+
+        # Test when authentication is required but not provided
+        response = await self.adapter.on_message(mock_message)
+        self.assertEqual(response.response_code, 401)
+        mock_get_user.assert_not_called()
+
+    @patch.object(CleverThisServiceAdapter, "get_current_user")
+    @patch.object(CleverThisServiceAdapter, "get")
+    async def test_on_message_authenticated(self, mock_get, mock_get_user):
+        mock_message = MagicMock(spec=AMQMessage)
+        mock_message.headers = {"Authorization": ["Bearer valid_token"]}
+        mock_message.path = "/some/path"
+        mock_message.method = "GET"
+        mock_message.id = "123"
+        mock_message.trace_info = {}
+
+        mock_user = MagicMock()
+        mock_get_user.return_value = mock_user
+
+        mock_response = MagicMock(spec=DataResponse)
+        mock_get.return_value = mock_response
+
+        response = await self.adapter.on_message(mock_message)
+
+        self.assertEqual(response, mock_response)
+        mock_get_user.assert_called_once_with("valid_token")
+        mock_get.assert_called_once_with(mock_message, mock_user)
+
+    @patch.object(CleverThisServiceAdapter, "get_current_user")
+    async def test_on_message_login_path(self, mock_get_user):
+        mock_message = MagicMock(spec=AMQMessage)
+        mock_message.headers = {}
+        mock_message.path = "/login"
+        mock_message.method = "POST"
+        mock_message.id = "123"
+        mock_message.trace_info = {}
+
+        mock_response = MagicMock(spec=DataResponse)
+        with patch.object(
+            self.adapter, "post", return_value=mock_response
+        ) as mock_post:
+            response = await self.adapter.on_message(mock_message)
+            self.assertEqual(response, mock_response)
+            mock_get_user.assert_not_called()
+            mock_post.assert_called_once()
+
+    @patch.object(CleverThisServiceAdapter, "get_current_user")
+    async def test_on_message_all_methods(self, mock_get_user):
+        mock_user = MagicMock()
+        mock_get_user.return_value = mock_user
+
+        mock_response = MagicMock(spec=DataResponse)
+
+        methods = ["GET", "PUT", "POST", "DELETE", "HEAD"]
+        for method in methods:
+            with self.subTest(method=method):
+                mock_message = MagicMock(spec=AMQMessage)
+                mock_message.headers = {"Authorization": ["Bearer valid_token"]}
+                mock_message.path = "/some/path"
+                mock_message.method = method
+                mock_message.id = "123"
+                mock_message.trace_info = {}
+
+                with patch.object(
+                    self.adapter, method.lower(), return_value=mock_response
+                ) as mock_method:
+                    response = await self.adapter.on_message(mock_message)
+                    self.assertEqual(response, mock_response)
+                    mock_method.assert_called_once()
+
+    async def test_get_with_session(self):
+        mock_message = MagicMock(spec=DataMessage)
+        mock_message.path = "/api/test"
+        mock_message.headers = {"X-Test": "value"}
+        mock_message.trace_info = {"trace-id": "123"}
+        mock_message.id = "msg123"
+
+        mock_response = MagicMock(spec=ClientResponse)
+        mock_response.status = 200
+        mock_response.content_type = "application/json"
+        mock_response.read = AsyncMock(return_value=b'{"result": "success"}')
+
+        self.mock_session.get.return_value = mock_response
+
+        response = await self.adapter.get(mock_message, None)
+
+        self.assertEqual(response.response_code, 200)
+        self.mock_session.get.assert_called_once_with(
+            "http://testhost:8080/api/test",
+            headers={"X-Test": "value", "trace-id": "123"},
+        )
+
+    async def test_get_without_session(self):
+        self.adapter.session = None
+
+        mock_message = MagicMock(spec=DataMessage)
+        mock_message.path = "/api/test"
+        mock_message.id = "msg123"
+        mock_message.trace_info = {}
+
+        response = await self.adapter.get(mock_message, None)
+
+        self.assertEqual(response.response_code, 404)
+
+    async def test_put_with_form_handling(self):
+        mock_message = MagicMock(spec=DataMessage)
+        mock_message.path = "/api/test"
+        mock_message.headers = {}
+        mock_message.trace_info = {}
+        mock_message.id = "msg123"
+        mock_message.body = b"test=value"
+
+        mock_response = MagicMock(spec=ClientResponse)
+        mock_response.status = 200
+        mock_response.content_type = "application/json"
+        mock_response.read = AsyncMock(return_value=b'{"result": "success"}')
+
+        self.mock_session.put.return_value = mock_response
+
+        with patch.object(
+            self.adapter,
+            "handle_possible_form",
+            wraps=self.adapter.handle_possible_form,
+        ) as mock_handler:
+            response = await self.adapter.put(mock_message, None)
+
+            self.assertEqual(response.response_code, 200)
+            mock_handler.assert_called_once()
+
+    async def test_post_with_form_handling(self):
+        mock_message = MagicMock(spec=DataMessage)
+        mock_message.path = "/api/test"
+        mock_message.headers = {}
+        mock_message.trace_info = {}
+        mock_message.id = "msg123"
+        mock_message.body = b"test=value"
+
+        mock_response = MagicMock(spec=ClientResponse)
+        mock_response.status = 201
+        mock_response.content_type = "application/json"
+        mock_response.read = AsyncMock(return_value=b'{"result": "created"}')
+
+        self.mock_session.post.return_value = mock_response
+
+        with patch.object(
+            self.adapter,
+            "handle_possible_form",
+            wraps=self.adapter.handle_possible_form,
+        ) as mock_handler:
+            response = await self.adapter.post(mock_message, None)
+
+            self.assertEqual(response.response_code, 201)
+            mock_handler.assert_called_once()
+
+    async def test_head_not_implemented(self):
+        mock_message = MagicMock(spec=DataMessage)
+        mock_message.id = "msg123"
+
+        with self.assertRaises(NotImplementedError):
+            await self.adapter.head(mock_message, None)
+
+    async def test_delete_not_implemented(self):
+        mock_message = MagicMock(spec=DataMessage)
+        mock_message.id = "msg123"
+
+        # Assuming delete is not implemented in base class
+        with self.assertRaises(NotImplementedError):
+            await self.adapter.delete(mock_message, None)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/adapter/test_data_message_factory.py

@@ -0,0 +1,129 @@
+from datetime import datetime, timezone
+from unittest import TestCase
+
+from amqp.adapter.data_message_factory import DataMessageFactory
+
+
+class DataMessageFactoryTest(TestCase):
+    def test_from_bytes(self):
+        raw = "\002\027A64f3b5af4d00000a4000|m=POST|d=clever-pybook.localhost|p=/debug|h={Accept=[*/*],X-Forwarded-Host=[clever-pybook.localhost:8085],User-Agent=[curl/8.7.1],X-Forwarded-Proto=[http],X-Forwarded-For=[172.18.0.1],Host=[clever-pybook.localhost:8085],Accept-Encoding=[gzip],Content-Length=[1475],X-Forwarded-Port=[8085],X-Real-Ip=[172.18.0.1],X-Forwarded-Server=[c5945bef67d1],Content-Type=[multipart/form-data; boundary=------------------------j1RjIqLrFH07XMsyMvpS1v]}|t={traceparent=00-bed6eb7de0d8ccf4c80b145a4bfec357-8d1f3bafc471bf0f-01}|b=TEST_ME"
+        msg = DataMessageFactory.from_bytes(raw.encode("utf-8"))
+        self.assertIsNotNone(msg.id)
+        self.assertEqual(
+            msg.id.as_string(),
+            "64F3B5AF4D00000A4000".lower(),
+            "SnowflakeID incorrectly parsed.",
+        )
+        self.assertEqual(
+            msg.headers["Content-Type"][0],
+            "multipart/form-data; boundary=------------------------j1RjIqLrFH07XMsyMvpS1v",
+        )
+
+    def test_generate_snowflake_id(self):
+        _id = DataMessageFactory.get_instance(1).generate_snowflake_id()
+        self.assertEqual(
+            "00001000", _id.as_string()[-8:], "Instance ID not set correctly"
+        )
+
+        _id = DataMessageFactory(2).generate_snowflake_id()
+        self.assertEqual(
+            "00002000", _id.as_string()[-8:], "Instance ID not set correctly"
+        )
+
+        _f = DataMessageFactory(127)
+        _i0 = _f.generate_snowflake_id()
+        _i1 = _f.generate_snowflake_id()
+        _i2 = _f.generate_snowflake_id()
+        _i3 = _f.generate_snowflake_id()
+        self.assertEqual(
+            "0007F000".lower(),
+            _i0.as_string()[-8:],
+            "Instance ID and sequence not set correctly",
+        )
+        self.assertEqual(
+            "0007F001".lower(),
+            _i1.as_string()[-8:],
+            "Instance ID and sequence not set correctly",
+        )
+        self.assertEqual(
+            "0007F002".lower(),
+            _i2.as_string()[-8:],
+            "Instance ID and sequence not set correctly",
+        )
+        self.assertEqual(
+            "0007F003".lower(),
+            _i3.as_string()[-8:],
+            "Instance ID and sequence not set correctly",
+        )
+
+    def test_create_request_message(self):
+        _f = DataMessageFactory(34)
+        _req = _f.create_request_message(
+            "PUT",
+            "localhost",
+            "/test/me",
+            {"k1": ["v1"], "k2": ["v2"]},
+            "Quick brown fox",
+        )
+        self.assertEqual("localhost", _req.domain, "Domain not set correctly")
+        self.assertEqual("/test/me", _req.path, "Context path not set correctly")
+        self.assertEqual("v1", _req.headers["k1"][0], "Headers not set correctly")
+        self.assertEqual(b"Quick brown fox", _req.body(), "Body not correctly decoded")
+
+    def test_serialize(self):
+        _f = DataMessageFactory(34)
+        _req = _f.create_request_message(
+            "PUT",
+            "localhost",
+            "/test/me",
+            {"k1": ["v1"], "k2": ["v2"]},
+            "Quick brown fox",
+        )
+        _ser = _f.serialize(_req)
+        self.assertEqual(65, _ser[2])
+        _deser = _f.from_bytes(_ser)
+        self.assertEqual(_req.id.as_string(), _deser.id.as_string())
+        self.assertEqual("localhost", _deser.domain)
+        self.assertEqual("/test/me", _deser.path)
+        self.assertEqual(b"Quick brown fox", _deser.body())
+        self.assertEqual("v2", _deser.headers["k2"][0])
+
+    def test_amq_message_routing_key(self):
+        _f = DataMessageFactory(35)
+        _req = _f.create_request_message(
+            "PUT",
+            "localhost",
+            "/test/me",
+            {"k1": ["v1"], "k2": ["v2"]},
+            "Quick brown fox",
+        )
+        _key = _f.amq_message_routing_key(_req)
+        self.assertEqual(
+            "localhost.test.me",
+            _key,
+            "Message routing key wrongly extracted from the message",
+        )
+
+    def test_get_timestamp_from_id(self):
+        _f = DataMessageFactory(35)
+        _timestamp = _f.get_timestamp_from_id(_f.generate_snowflake_id())
+        _x_timestamp = int(datetime.now(timezone.utc).timestamp() * 1000)
+        self.assertLessEqual(
+            _x_timestamp - _timestamp,
+            1,
+            "Timestamp in SnowflakeId differs too much from current time",
+        )
+
+    def test_to_string(self):
+        _f = DataMessageFactory(35)
+        _req = _f.create_request_message(
+            "PUT",
+            "localhost",
+            "/test/me",
+            {"k1": ["v1"], "k2": ["v2"]},
+            "Quick brown fox",
+        )
+        _as_str = _f.to_string(_req)
+        self.assertGreater(
+            _as_str.index("domain='localhost'"), 0, "Expected content not found"
+        )

tests/adapter/test_data_response_factory.py

@@ -0,0 +1,40 @@
+from unittest import TestCase
+
+from amqp.adapter.data_message_factory import DataMessageFactory
+from amqp.adapter.data_response_factory import DataResponseFactory
+from amqp.model.snowflake_id import SnowflakeId
+
+
+class TestDataResponseFactory(TestCase):
+
+    def test_create_async_response_message(self):
+        _f = DataResponseFactory()
+        _g = DataMessageFactory(111)
+        _resp = _f.create_async_response_message(_g.generate_snowflake_id())
+        self.assertEqual(200, _resp.response_code)
+        self.assertEqual(b"OK", _resp.response)
+
+    def test_serialize(self):
+        _f = DataResponseFactory()
+        _g = DataMessageFactory(111)
+        _snowflakeId: SnowflakeId = _g.generate_snowflake_id()
+        _resp = _f.create_async_response_message(_snowflakeId)
+        _ser = _f.serialize(_resp)
+        self.assertEqual(
+            2,
+            str(_ser[2:]).index(_f.MAGIC_BYTE_HTTP_REQUEST + _snowflakeId.as_string()),
+            "Magic byte and SnowflakeId not found",
+        )
+        self.assertGreater(str(_ser[2:]).index("|e=|"), 2)
+        self.assertEqual((_ser[0] + _ser[1]), len(_ser) - 2 - len("T0s="))
+
+    def test_from_bytes(self):
+        _f = DataResponseFactory()
+        _g = DataMessageFactory(111)
+        _snowflakeId: SnowflakeId = _g.generate_snowflake_id()
+        _resp = _f.create_async_response_message(_snowflakeId)
+        _ser = _f.serialize(_resp)
+        _deser = _f.from_bytes(_ser)
+        self.assertEqual(_snowflakeId.as_string(), _deser.id)
+        self.assertEqual(200, _deser.response_code)
+        self.assertEqual(b"OK", _deser.response)

tests/adapter/test_file_handler.py

@@ -0,0 +1,50 @@
+import os
+import tempfile
+from unittest import TestCase
+
+from amqp.adapter.file_handler import StreamingFileHandler
+
+
+class TestFileHandler(TestCase):
+    def test_ensure_directory_exists(self):
+        """
+        Test function for ensure_directory_exists.  Creates a temporary directory
+        and files to test the versioning logic.
+        """
+
+        # Create a temporary directory
+        file_handler = StreamingFileHandler()
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Test case 1: File does not exist
+            filepath1 = os.path.join(temp_dir, "test1.txt")
+            result1 = file_handler.ensure_directory_exists(filepath1)
+            assert result1 == filepath1, f"Test Case 1 Failed: {result1} != {filepath1}"
+            # Create the file
+            with open(filepath1, "w") as f:
+                f.write("test")
+
+            # Test case 2: File exists, should create versioned file
+            filepath2 = os.path.join(temp_dir, "test2.txt")
+            result2 = file_handler.ensure_directory_exists(filepath2)
+            assert result2 == filepath2, f"Test Case 2 Failed: {result2} != {filepath2}"
+            with open(filepath2, "w") as f:
+                f.write("test")
+            result2_v1 = file_handler.ensure_directory_exists(filepath2)
+            assert result2_v1 == os.path.join(
+                temp_dir, "test2.1.txt"
+            ), f"Test Case 2 Version 1 Failed: {result2_v1} != {os.path.join(temp_dir, 'test2.1.txt')}"
+
+            # Create a few versioned files
+            with open(os.path.join(temp_dir, "test3.txt"), "w") as f:
+                f.write("test")
+            with open(os.path.join(temp_dir, "test3.1.txt"), "w") as f:
+                f.write("test")
+            with open(os.path.join(temp_dir, "test3.2.txt"), "w") as f:
+                f.write("test")
+            filepath3 = os.path.join(temp_dir, "test3.txt")
+            result3_v3 = file_handler.ensure_directory_exists(filepath3)
+            assert result3_v3 == os.path.join(
+                temp_dir, "test3.3.txt"
+            ), f"Test Case 3 Version 3 Failed: {result3_v3} != {os.path.join(temp_dir, 'test3.3.txt')}"
+
+            print("All test cases passed!")

tests/adapter/test_pydantic_serializer.py

@@ -0,0 +1,38 @@
+from unittest import TestCase
+
+from amqp.adapter.pydantic_serializer import parse_url_query
+
+
+class PydanticSerializerTest(TestCase):
+    def test_serialize_object(self):
+        assert True
+
+    def test_deserialize_object(self):
+        assert True
+
+    def test__convert_value(self):
+        assert True
+
+    def test__is_uuid_string(self):
+        assert True
+
+    def test__is_datetime_string(self):
+        assert True
+
+    def test__is_int_string(self):
+        assert True
+
+    def test_serialize_to_json(self):
+        assert True
+
+    def test_parse_url_query(self):
+        path, args = parse_url_query(
+            "http://localhost:8080/api/v0/unstructured/863e4d9f-4612-4173-8d0a-3de34cc7a741?response_type=JsonFile"
+        )
+        assert path == "/api/v0/unstructured/863e4d9f-4612-4173-8d0a-3de34cc7a741"
+        assert args == {"response_type": "JsonFile"}
+        path, args = parse_url_query(
+            "http://localhost:8080/api/v0/unstructured/863e4d9f-4612-4173-8d0a-3de34cc7a741"
+        )
+        assert path == "/api/v0/unstructured/863e4d9f-4612-4173-8d0a-3de34cc7a741"
+        assert args == {}

tests/adapter/test_service_message_factory.py

@@ -0,0 +1,59 @@
+from unittest import TestCase
+
+from amqp.adapter.service_message_factory import (
+    ServiceMessageFactory,
+    RS,
+    INVALID_MESSAGE,
+)
+from amqp.model.model import ServiceMessage
+from amqp.model.service_message_type import ServiceMessageType
+
+
+class ServiceMessageTest(TestCase):
+
+    def test_to_string(self):
+        _f: ServiceMessageFactory = ServiceMessageFactory("bumble-bee")
+        message: ServiceMessage = _f.of(
+            ServiceMessageType.ROUTING_DATA, "Duck", "Donald"
+        )
+        message_str = _f.to_string(message)
+        self.assertTrue("|type=01|" in message_str)
+        self.assertTrue("|body=Duck" in message_str)
+        message: ServiceMessage = _f.of(
+            ServiceMessageType.ROUTING_DATA_REPLACE, "Duck", "Donald"
+        )
+        message_str = _f.to_string(message)
+        self.assertTrue("|type=03|" in message_str)
+        self.assertTrue("|body=Duck" in message_str)
+        self.assertTrue("|replyTo=bumble-bee" in message_str)
+        nullMsg = _f.to_string(None)
+        self.assertEqual("null", nullMsg)
+
+    def test_from(self):
+        _f: ServiceMessageFactory = ServiceMessageFactory("amqp-router")
+        message = _f.of(ServiceMessageType.ROUTING_DATA, "Duck", "Donald")
+        serialized = _f.serialize(message)
+        deserialized = _f.from_bytes(serialized)
+        assert deserialized.is_valid()
+        assert deserialized.message_type == ServiceMessageType.ROUTING_DATA
+        assert deserialized.recipient_name == "Donald"
+        assert deserialized.message == "Duck"
+        assert deserialized.reply_to == "amqp-router"
+        invalidMsg = _f.from_bytes(b"wrong")
+        self.assertEqual(invalidMsg, INVALID_MESSAGE)
+
+    def test_base64(self):
+        _f: ServiceMessageFactory = ServiceMessageFactory("amqp-router")
+        message = _f.as_base64(ServiceMessageType.ROUTING_DATA, "Duck", "Donald")
+        serialized = _f.serialize(message)
+        assert b"|81|" in serialized
+        deserialized = _f.from_bytes(serialized)
+        assert deserialized.is_valid()
+        assert deserialized.message_type == ServiceMessageType.ROUTING_DATA
+        assert deserialized.message == "Duck"
+
+    def test_serialize(self):
+        _f: ServiceMessageFactory = ServiceMessageFactory("amqp-router")
+        message = _f.of(ServiceMessageType.ROUTING_DATA, "Duck", "Donald")
+        serialized = _f.serialize(message)
+        assert RS + "0" + str(ServiceMessageType.ROUTING_DATA.value)