Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
9543b2c
feat: tortoise
faviansamatha Nov 24, 2025
5fcf8ef
Add thread pool executors
faviansamatha Dec 1, 2025
c2fcb0f
fix logs
faviansamatha Dec 2, 2025
11a44ae
add sql alchemy tortoise connection provider
faviansamatha Dec 3, 2025
f9958da
add tortoise integration tests
faviansamatha Dec 3, 2025
e9d59db
Added more intergation tests and fixed unit tests
faviansamatha Dec 4, 2025
ab5fdf4
fix formatting
faviansamatha Dec 5, 2025
82d109e
fix static check
faviansamatha Dec 6, 2025
300e6af
rename path to backends instead of backend
faviansamatha Dec 6, 2025
cb67cdb
temp: run integration tests on this branch
faviansamatha Dec 6, 2025
d2f7e62
refactor async wrappers
faviansamatha Dec 8, 2025
b83c89f
add async connection pool
faviansamatha Dec 11, 2025
a67c200
Added custom pool
faviansamatha Dec 17, 2025
383679c
Removed sqlalchemytortoiseprovider
faviansamatha Dec 17, 2025
2053cb0
add tests for async wrapper and connection pool
faviansamatha Dec 17, 2025
6578b7d
fix static checks
faviansamatha Dec 18, 2025
85b4ab3
move weight random host selector test to unit tests
faviansamatha Dec 18, 2025
93d720e
Add integration test to branch
faviansamatha Dec 18, 2025
55d4b1c
fix static check
faviansamatha Dec 18, 2025
e629cc8
revert unecessary formatting changes
faviansamatha Dec 18, 2025
b98d0f9
integ test debug
faviansamatha Dec 18, 2025
117f2ce
remove iam
faviansamatha Dec 18, 2025
4b955e4
run a subset of integration tests. Only mysql with test_tortoise filter
faviansamatha Dec 18, 2025
3ce3fcc
tmate
faviansamatha Dec 18, 2025
0b3b1c0
Run test without outputting to console
faviansamatha Dec 19, 2025
fea8788
remove aurora connection tracker from plugin list
faviansamatha Dec 19, 2025
0bedc08
Added monitoring thread for docker container
faviansamatha Dec 19, 2025
66d8b74
add pytest timeout
faviansamatha Dec 19, 2025
278299a
test removing is_closed check to avoid deadlock
faviansamatha Dec 19, 2025
42292bf
retest
faviansamatha Dec 19, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 14 additions & 3 deletions .github/workflows/integration_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ on:
push:
branches:
- main
- feat/tortoise

permissions:
id-token: write # This is required for requesting the JWT
Expand All @@ -17,8 +18,9 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: [ "3.11", "3.12", "3.13" ]
environment: [ "mysql", "pg" ]
python-version: [ "3.11", "3.12", "3.13"]
# environment: [ "mysql", "pg" ]
environment: [ "mysql",]

steps:
- name: 'Clone repository'
Expand Down Expand Up @@ -48,8 +50,10 @@ jobs:
aws-region: ${{ secrets.AWS_DEFAULT_REGION }}

- name: 'Run LTS Integration Tests'
timeout-minutes: 120
run: |
./gradlew --no-parallel --no-daemon test-python-${{ matrix.python-version }}-${{ matrix.environment }} --info
./gradlew --no-parallel --no-daemon test-python-${{ matrix.python-version }}-${{ matrix.environment }} --info \
2>&1 | tee test-output.log
env:
RDS_CLUSTER_DOMAIN: ${{ secrets.DB_CONN_SUFFIX }}
RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
Expand All @@ -64,6 +68,13 @@ jobs:
path: ./tests/integration/container/reports
retention-days: 5

- name: Upload logs
uses: actions/upload-artifact@v4
if: always()
with:
name: pytest-integration-${{ matrix.python-version }}-${{ matrix.environment }}-log
path: test-output.log

latest-integration-tests:
name: Run Latest Integration Tests
runs-on: ubuntu-latest
Expand Down
9 changes: 6 additions & 3 deletions aws_advanced_python_wrapper/custom_endpoint_plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,8 @@ def _run(self):
len(endpoints),
endpoint_hostnames)

sleep(self._refresh_rate_ns / 1_000_000_000)
if self._stop_event.wait(self._refresh_rate_ns / 1_000_000_000):
break
continue

endpoint_info = CustomEndpointInfo.from_db_cluster_endpoint(endpoints[0])
Expand All @@ -178,7 +179,8 @@ def _run(self):
if cached_info is not None and cached_info == endpoint_info:
elapsed_time = perf_counter_ns() - start_ns
sleep_duration = max(0, self._refresh_rate_ns - elapsed_time)
sleep(sleep_duration / 1_000_000_000)
if self._stop_event.wait(sleep_duration / 1_000_000_000):
break
continue

logger.debug(
Expand All @@ -196,7 +198,8 @@ def _run(self):

elapsed_time = perf_counter_ns() - start_ns
sleep_duration = max(0, self._refresh_rate_ns - elapsed_time)
sleep(sleep_duration / 1_000_000_000)
if self._stop_event.wait(sleep_duration / 1_000_000_000):
break
continue
except InterruptedError as e:
raise e
Expand Down
28 changes: 28 additions & 0 deletions aws_advanced_python_wrapper/errors.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,3 +45,31 @@ class FailoverSuccessError(FailoverError):

class ReadWriteSplittingError(AwsWrapperError):
__module__ = "aws_advanced_python_wrapper"


class AsyncConnectionPoolError(AwsWrapperError):
__module__ = "aws_advanced_python_wrapper"


class PoolNotInitializedError(AsyncConnectionPoolError):
__module__ = "aws_advanced_python_wrapper"


class PoolClosingError(AsyncConnectionPoolError):
__module__ = "aws_advanced_python_wrapper"


class PoolExhaustedError(AsyncConnectionPoolError):
__module__ = "aws_advanced_python_wrapper"


class ConnectionReleasedError(AsyncConnectionPoolError):
__module__ = "aws_advanced_python_wrapper"


class PoolSizeLimitError(AsyncConnectionPoolError):
__module__ = "aws_advanced_python_wrapper"


class PoolHealthCheckError(AsyncConnectionPoolError):
__module__ = "aws_advanced_python_wrapper"
2 changes: 1 addition & 1 deletion aws_advanced_python_wrapper/iam_plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ def _connect(self, host_info: HostInfo, props: Properties, connect_func: Callabl

is_cached_token = (token_info is not None and not token_info.is_expired())
if not self._plugin_service.is_login_exception(error=e) or not is_cached_token:
raise AwsWrapperError(Messages.get_formatted("IamAuthPlugin.ConnectException", e)) from e
raise
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We need to remove the wrapped error. If we get a failover and we get a network error, the failover plugin will not detect it.


# Login unsuccessful with cached token
# Try to generate a new token and try to connect again
Expand Down
6 changes: 4 additions & 2 deletions aws_advanced_python_wrapper/plugin_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -703,8 +703,7 @@ def is_plugin_in_use(self, plugin_class: Type[Plugin]) -> bool:

def release_resources(self):
try:
if self.current_connection is not None and not self.driver_dialect.is_closed(
self.current_connection):
if self.current_connection is not None:
self.current_connection.close()
except Exception:
# ignore
Expand Down Expand Up @@ -845,6 +844,9 @@ def get_plugins(self) -> List[Plugin]:
if plugin_codes is None:
plugin_codes = WrapperProperties.DEFAULT_PLUGINS

if plugin_codes.lower() == "none":
plugin_codes = ""

if plugin_codes != "":
plugin_factories = self.create_plugin_factories_from_list(plugin_codes.split(","))

Expand Down
47 changes: 47 additions & 0 deletions aws_advanced_python_wrapper/tortoise_orm/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from tortoise.backends.base.config_generator import DB_LOOKUP


def cast_to_bool(value):
"""Generic function to cast various types to boolean."""
if isinstance(value, bool):
return value
if isinstance(value, str):
return value.lower() in ('true', '1', 'yes', 'on')
return bool(value)


# Register AWS MySQL backend
DB_LOOKUP["aws-mysql"] = {
"engine": "aws_advanced_python_wrapper.tortoise_orm.backends.mysql",
"vmap": {
"path": "database",
"hostname": "host",
"port": "port",
"username": "user",
"password": "password",
},
"defaults": {"port": 3306, "charset": "utf8mb4", "sql_mode": "STRICT_TRANS_TABLES"},
"cast": {
"minsize": int,
"maxsize": int,
"connect_timeout": int,
"echo": cast_to_bool,
"use_unicode": cast_to_bool,
"ssl": cast_to_bool,
"use_pure": cast_to_bool
},
}
13 changes: 13 additions & 0 deletions aws_advanced_python_wrapper/tortoise_orm/async_support/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Loading