From 32a492ca373a9673c9acbda668f8e1410224ddd7 Mon Sep 17 00:00:00 2001 From: Jay Chung Date: Sun, 27 Apr 2025 12:09:09 +0800 Subject: [PATCH 01/13] CI: update os version --- .github/workflows/ci.yaml | 11 +++++------ setup.cfg | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9d1b94e..4d281f6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -77,22 +77,21 @@ jobs: # YAML parse `3.10` to `3.1`, so we have to add quotes for `'3.10'`, see also: # https://github.com/actions/setup-python/issues/160#issuecomment-724485470 python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] - # FIXME: temp change os to ubuntu-20.04 to fix python can not found error https://github.com/actions/setup-python/issues/162#issuecomment-1325307787 - os: [ubuntu-20.04, macOS-latest, windows-latest] + os: [ubuntu-latest, macOS-latest, windows-latest] exclude: # Skip because dependence [py4j](https://pypi.org/project/py4j/) not work on those environments - os: windows-latest python-version: '3.10' - os: windows-latest - python-version: 3.11 + python-version: '3.11' - os: windows-latest - python-version: 3.12 + python-version: '3.12' # Python 3.9 is on macos-13 but not macos-latest (macos-14-arm64) # https://github.com/actions/setup-python/issues/696#issuecomment-1637587760 - os: macos-latest - python-version: 3.9 + python-version: '3.9' include: - - python-version: 3.9 + - python-version: '3.9' os: macos-13 steps: - uses: actions/checkout@v3 diff --git a/setup.cfg b/setup.cfg index 992e2cb..34629b3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,7 @@ classifiers = Development Status :: 4 - Beta Environment :: Console Intended Audience :: Developers - License :: OSI Approved :: Apache Software License + License :: Apache-2.0 Operating System :: Unix Operating System :: POSIX Operating System :: Microsoft :: Windows From 3142e2892286e9978d0f0c0f19a15e8e1b5a562a Mon Sep 17 00:00:00 2001 From: Jay Chung Date: Sun, 27 Apr 2025 12:15:19 +0800 Subject: [PATCH 02/13] only build ds standalone image [run-it] --- .github/workflows/ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4d281f6..c3637ec 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -200,7 +200,8 @@ jobs: ./mvnw -B clean install \ -Dmaven.test.skip=true \ -Dspotless.skip=true \ - -Pdocker,staging -Ddocker.tag=ci + -Pdocker,staging -Ddocker.tag=ci \ + -pl dolphinscheduler-standalone-server -am - name: Set up Python uses: actions/setup-python@v4 with: From 5a0ab279624c53e38e758cce8a67f9b660dc1bb9 Mon Sep 17 00:00:00 2001 From: Jay Chung Date: Sun, 27 Apr 2025 12:28:17 +0800 Subject: [PATCH 03/13] revert docker images [run-it] --- .github/workflows/ci.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c3637ec..52e2362 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -209,6 +209,9 @@ jobs: - name: Install Dependences run: | python -m pip install --upgrade ${{ env.DEPENDENCES }} + - name: check docker images + run: | + docker images - name: Run Integrate Tests working-directory: src run: | From 241d707bc22d1effc46283f196eaa772f7fad10e Mon Sep 17 00:00:00 2001 From: Jay Chung Date: Sun, 27 Apr 2025 12:40:14 +0800 Subject: [PATCH 04/13] docker images not exists [run-it] --- .github/workflows/ci.yaml | 3 +-- setup.cfg | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 52e2362..818dcbc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -200,8 +200,7 @@ jobs: ./mvnw -B clean install \ -Dmaven.test.skip=true \ -Dspotless.skip=true \ - -Pdocker,staging -Ddocker.tag=ci \ - -pl dolphinscheduler-standalone-server -am + -Pdocker,staging -Ddocker.tag=ci - name: Set up Python uses: actions/setup-python@v4 with: diff --git a/setup.cfg b/setup.cfg index 34629b3..e76e8ea 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,9 +24,9 @@ long_description = file: README.md long_description_content_type = text/markdown author = Apache Software Foundation author_email = dev@dolphinscheduler.apache.org -license = Apache License 2.0 +license = Apache-2.0 license_files = - file: LICENSE + LICEN[CS]E* keywords = dolphinscheduler workflow @@ -37,7 +37,6 @@ classifiers = Development Status :: 4 - Beta Environment :: Console Intended Audience :: Developers - License :: Apache-2.0 Operating System :: Unix Operating System :: POSIX Operating System :: Microsoft :: Windows From aec59e13a276e27e0af653b140b958c5f58c24f0 Mon Sep 17 00:00:00 2001 From: Jay Chung Date: Sun, 27 Apr 2025 12:55:22 +0800 Subject: [PATCH 05/13] get docker logs [run-it] --- .github/workflows/ci.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 818dcbc..9c59775 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -208,13 +208,14 @@ jobs: - name: Install Dependences run: | python -m pip install --upgrade ${{ env.DEPENDENCES }} - - name: check docker images - run: | - docker images - name: Run Integrate Tests working-directory: src run: | python -m tox -vv -e integrate-test + - name: check docker log + run: | + docker ps + docker logs ci-dolphinscheduler-standalone-server result: name: CI if: always() From 2087c9ca2f5676178c90a78b5f898838ff79c62e Mon Sep 17 00:00:00 2001 From: Jay Chung Date: Sun, 27 Apr 2025 12:56:40 +0800 Subject: [PATCH 06/13] do not remove container [run-it] --- tests/integration/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 7a962ef..b83ffcf 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -49,4 +49,4 @@ def docker_setup_teardown(): ) assert container is not None yield - docker_wrapper.remove_container() + # docker_wrapper.remove_container() From 9eee4c1015898a349b62eadc5c49b246bca72b33 Mon Sep 17 00:00:00 2001 From: Jay Chung Date: Sun, 27 Apr 2025 13:25:53 +0800 Subject: [PATCH 07/13] get container log [run-it] --- tests/testing/docker_wrapper.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/testing/docker_wrapper.py b/tests/testing/docker_wrapper.py index 8f1a9fb..43fc343 100644 --- a/tests/testing/docker_wrapper.py +++ b/tests/testing/docker_wrapper.py @@ -72,7 +72,7 @@ def run_until_log( log_byte = str.encode(log) container = self.run(*args, **kwargs) - timeout_threshold = 10 * 60 + timeout_threshold = 2 * 60 start_time = time.time() while time.time() <= start_time + timeout_threshold: if log_byte in container.logs(tail=1000): @@ -80,10 +80,11 @@ def run_until_log( time.sleep(2) # Stop container and raise error when reach timeout threshold but do not appear specific log output else: + container_log = container.logs() container.remove(force=True) raise RuntimeError( - "Can not capture specific log `%s` in %d seconds, remove container.", - (log, timeout_threshold), + "Can not capture specific log `%s` in %d seconds, remove container, the whole log is:\n%s", + (log, timeout_threshold, container_log), ) return container From e49add592887a247178f30098056515c82faf42b Mon Sep 17 00:00:00 2001 From: Jay Chung Date: Sun, 27 Apr 2025 13:41:15 +0800 Subject: [PATCH 08/13] show all log [run-it] --- tests/testing/docker_wrapper.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/testing/docker_wrapper.py b/tests/testing/docker_wrapper.py index 43fc343..9e32196 100644 --- a/tests/testing/docker_wrapper.py +++ b/tests/testing/docker_wrapper.py @@ -23,6 +23,9 @@ import docker from docker.errors import ImageNotFound from docker.models.containers import Container +import logging + +logger = logging.getLogger(__name__) class DockerWrapper: @@ -72,7 +75,7 @@ def run_until_log( log_byte = str.encode(log) container = self.run(*args, **kwargs) - timeout_threshold = 2 * 60 + timeout_threshold = 5 * 60 start_time = time.time() while time.time() <= start_time + timeout_threshold: if log_byte in container.logs(tail=1000): @@ -81,10 +84,11 @@ def run_until_log( # Stop container and raise error when reach timeout threshold but do not appear specific log output else: container_log = container.logs() + logger.error(container_log.decode('utf-8')) container.remove(force=True) raise RuntimeError( - "Can not capture specific log `%s` in %d seconds, remove container, the whole log is:\n%s", - (log, timeout_threshold, container_log), + "Can not capture specific log `%s` in %d seconds, remove container.", + (log, timeout_threshold), ) return container From e035d636e666fc2edb7ffd6eb6f31f5462cc840e Mon Sep 17 00:00:00 2001 From: Jay Chung Date: Sun, 27 Apr 2025 13:47:48 +0800 Subject: [PATCH 09/13] style [run-it] --- tests/testing/docker_wrapper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/testing/docker_wrapper.py b/tests/testing/docker_wrapper.py index 9e32196..108a6e8 100644 --- a/tests/testing/docker_wrapper.py +++ b/tests/testing/docker_wrapper.py @@ -18,12 +18,12 @@ """Wrap docker commands for easier create docker container.""" from __future__ import annotations +import logging import time import docker from docker.errors import ImageNotFound from docker.models.containers import Container -import logging logger = logging.getLogger(__name__) @@ -84,7 +84,7 @@ def run_until_log( # Stop container and raise error when reach timeout threshold but do not appear specific log output else: container_log = container.logs() - logger.error(container_log.decode('utf-8')) + logger.error(container_log.decode("utf-8")) container.remove(force=True) raise RuntimeError( "Can not capture specific log `%s` in %d seconds, remove container.", From aa3476857a169a27deca61bb770b22c0dea527cc Mon Sep 17 00:00:00 2001 From: Jay Chung Date: Sun, 27 Apr 2025 14:09:01 +0800 Subject: [PATCH 10/13] disable master and worker protection [run-it] --- tests/integration/conftest.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b83ffcf..9ec863c 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -41,7 +41,11 @@ def docker_setup_teardown(): docker_wrapper = DockerWrapper( image="apache/dolphinscheduler-standalone-server:ci", container_name="ci-dolphinscheduler-standalone-server", - environment={"API_PYTHON_GATEWAY_ENABLED": "true"}, + environment={ + "API_PYTHON_GATEWAY_ENABLED": "true", + "MASTER_SERVER_LOAD_PROTECTION_ENABLED": "false", + "WORKER_SERVER_LOAD_PROTECTION_ENABLED": "false", + }, ) ports = {"25333/tcp": 25333, "12345/tcp": 12345} container = docker_wrapper.run_until_log( From fe16a606cfdf3f8169da0b37b1b9ef3faa100b96 Mon Sep 17 00:00:00 2001 From: Jay Chung Date: Sun, 27 Apr 2025 14:12:31 +0800 Subject: [PATCH 11/13] add more log detail [run-it] --- tests/testing/docker_wrapper.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/testing/docker_wrapper.py b/tests/testing/docker_wrapper.py index 108a6e8..ce8ba0c 100644 --- a/tests/testing/docker_wrapper.py +++ b/tests/testing/docker_wrapper.py @@ -84,7 +84,13 @@ def run_until_log( # Stop container and raise error when reach timeout threshold but do not appear specific log output else: container_log = container.logs() + logger.error( + "Cannot find specific log `%s` in %d seconds, the whole log as below", + log, + timeout_threshold, + ) logger.error(container_log.decode("utf-8")) + container.remove(force=True) raise RuntimeError( "Can not capture specific log `%s` in %d seconds, remove container.", From 1e815b8335fca02789637f616fee70b5bd846cda Mon Sep 17 00:00:00 2001 From: Jay Chung Date: Sun, 27 Apr 2025 14:24:11 +0800 Subject: [PATCH 12/13] add show log into conftest [run-it] --- .github/workflows/ci.yaml | 4 ---- tests/integration/conftest.py | 10 +++++++++- tests/testing/docker_wrapper.py | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9c59775..4d281f6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -212,10 +212,6 @@ jobs: working-directory: src run: | python -m tox -vv -e integrate-test - - name: check docker log - run: | - docker ps - docker logs ci-dolphinscheduler-standalone-server result: name: CI if: always() diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 9ec863c..cf17afb 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -17,12 +17,15 @@ """py.test conftest.py file for package integration test.""" +import logging import os import pytest from tests.testing.docker_wrapper import DockerWrapper +logger = logging.getLogger(__name__) + @pytest.fixture(scope="package", autouse=True) def docker_setup_teardown(): @@ -53,4 +56,9 @@ def docker_setup_teardown(): ) assert container is not None yield - # docker_wrapper.remove_container() + container_logs = container.logs() + logger.info( + "Finished integration tests run, Container logs: %s", + container_logs.decode("utf-8"), + ) + docker_wrapper.remove_container() diff --git a/tests/testing/docker_wrapper.py b/tests/testing/docker_wrapper.py index ce8ba0c..866e365 100644 --- a/tests/testing/docker_wrapper.py +++ b/tests/testing/docker_wrapper.py @@ -75,7 +75,7 @@ def run_until_log( log_byte = str.encode(log) container = self.run(*args, **kwargs) - timeout_threshold = 5 * 60 + timeout_threshold = 6 * 60 start_time = time.time() while time.time() <= start_time + timeout_threshold: if log_byte in container.logs(tail=1000): From 2715a09838da232f349751f066a7feadb07c23c2 Mon Sep 17 00:00:00 2001 From: Jay Chung Date: Sun, 27 Apr 2025 14:44:14 +0800 Subject: [PATCH 13/13] fix it [run-it] --- tests/testing/docker_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testing/docker_wrapper.py b/tests/testing/docker_wrapper.py index 866e365..ce8ba0c 100644 --- a/tests/testing/docker_wrapper.py +++ b/tests/testing/docker_wrapper.py @@ -75,7 +75,7 @@ def run_until_log( log_byte = str.encode(log) container = self.run(*args, **kwargs) - timeout_threshold = 6 * 60 + timeout_threshold = 5 * 60 start_time = time.time() while time.time() <= start_time + timeout_threshold: if log_byte in container.logs(tail=1000):