From a2f07b7e6f1f02263d9142d53c6d5ee5571a8188 Mon Sep 17 00:00:00 2001 From: snowykami Date: Sat, 12 Oct 2024 23:35:01 +0800 Subject: [PATCH] :sparkles: first commit --- .github/workflows/pypi-publish.yml | 21 +++ .gitignore | 162 +++++++++++++++++++++++ croterline/__init__.py | 4 + croterline/context.py | 15 +++ croterline/process.py | 87 ++++++++++++ croterline/utils.py | 3 + pdm.lock | 206 +++++++++++++++++++++++++++++ pyproject.toml | 30 +++++ tests/test_subprocess.py | 34 +++++ 9 files changed, 562 insertions(+) create mode 100644 .github/workflows/pypi-publish.yml create mode 100644 .gitignore create mode 100644 croterline/__init__.py create mode 100644 croterline/context.py create mode 100644 croterline/process.py create mode 100644 croterline/utils.py create mode 100644 pdm.lock create mode 100644 pyproject.toml create mode 100644 tests/test_subprocess.py diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml new file mode 100644 index 0000000..402eae8 --- /dev/null +++ b/.github/workflows/pypi-publish.yml @@ -0,0 +1,21 @@ +name: Publish + +on: + push: + tags: + - 'v*' + +jobs: + pypi-publish: + name: upload release to PyPI + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v3 + + - uses: pdm-project/setup-pdm@v3 + + - name: Publish package distributions to PyPI + run: pdm publish diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..29c6762 --- /dev/null +++ b/.gitignore @@ -0,0 +1,162 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm-project.org/#use-with-ide +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +.idea/ diff --git a/croterline/__init__.py b/croterline/__init__.py new file mode 100644 index 0000000..0f67211 --- /dev/null +++ b/croterline/__init__.py @@ -0,0 +1,4 @@ +from multiprocessing import set_start_method + +# 设置Linux下默认开新进程的方式 +set_start_method("spawn", force=True) diff --git a/croterline/context.py b/croterline/context.py new file mode 100644 index 0000000..d414841 --- /dev/null +++ b/croterline/context.py @@ -0,0 +1,15 @@ +from typing import Callable, Any + +from magicoca.chan import Chan + + +class Context: + def __init__(self): + self.main_chan: Chan[Any] = Chan[Any]() # main to sub + self.sub_chan: Chan[Any] = Chan[Any]() # sub to main + + def set_value(self, key: str, value: Any): + setattr(self, key, value) + + def get_value(self, key: str) -> Any: + return getattr(self, key) diff --git a/croterline/process.py b/croterline/process.py new file mode 100644 index 0000000..f691744 --- /dev/null +++ b/croterline/process.py @@ -0,0 +1,87 @@ +from multiprocessing import Process as _Process +from typing import Callable, Any + +from croterline.context import Context +from croterline.utils import IsMainProcess + +type ProcessFuncType = Callable[[tuple[Any, ...], dict[str, Any]], None] + +_processes: dict[str, "SubProcess"] = {} + +_current_ctx: "Context | None" = None # 注入当前进程上下文 + + +class SubProcess: + def __init__( + self, name: str, func: ProcessFuncType, ctx: Context = Context, *args, **kwargs + ): + self.name = name + self.func = func + self.ctx = ctx + self.args = args + self.kwargs = kwargs + + self.process: _Process | None = None + + def start(self): + self.process = _Process( + target=_wrapper, + args=(self.func, self.ctx, *self.args), + kwargs=self.kwargs, + ) + self.process.start() + set_process(self.name, self) + + def terminate(self): + self.process.terminate() + set_process(self.name, None) + + def join(self, timeout: float = 0): + self.process.join(timeout=timeout) + set_process(self.name, None) + + +def set_process(name: str, process: SubProcess | None): + _processes[name] = process + + +def get_process(name: str) -> SubProcess | None: + """ + 获取进程对象,在主进程中调用,只可在主进程调用 + Args: + name: 进程名称 + Returns: + 进程对象 + """ + if not IsMainProcess: + raise RuntimeError( + "get_process with specific name can only be called in the main process." + ) + return _processes.get(name, None) + + +def get_ctx(name: str | None = None) -> Context | None: + """ + 获取进程上下文,在主进程中调用需指定进程名称,若在子进程中调用则无需指定进程名称 + Returns: + 进程上下文 + """ + if name is not None: + if not IsMainProcess: + raise RuntimeError( + "get_ctx with specific name can only be called in the main process." + ) + return _processes.get(name, None).ctx + else: + if IsMainProcess: + raise RuntimeError( + "get_ctx without name can only be called in the sub process." + ) + return _current_ctx + + +def _wrapper(func: ProcessFuncType, ctx: Context, *args, **kwargs): + global _current_ctx + + _current_ctx = ctx + func(*args, **kwargs) diff --git a/croterline/utils.py b/croterline/utils.py new file mode 100644 index 0000000..9e77642 --- /dev/null +++ b/croterline/utils.py @@ -0,0 +1,3 @@ +from multiprocessing import current_process + +IsMainProcess = current_process().name == "MainProcess" diff --git a/pdm.lock b/pdm.lock new file mode 100644 index 0000000..dc06f26 --- /dev/null +++ b/pdm.lock @@ -0,0 +1,206 @@ +# This file is @generated by PDM. +# It is not intended for manual editing. + +[metadata] +groups = ["default", "dev"] +strategy = ["inherit_metadata"] +lock_version = "4.5.0" +content_hash = "sha256:4affe52c3e672fd14a8c7a461b5a250cda491156c7673dd316492dc40f5e2512" + +[[metadata.targets]] +requires_python = ">=3.10" + +[[package]] +name = "black" +version = "24.10.0" +requires_python = ">=3.9" +summary = "The uncompromising code formatter." +groups = ["dev"] +dependencies = [ + "click>=8.0.0", + "mypy-extensions>=0.4.3", + "packaging>=22.0", + "pathspec>=0.9.0", + "platformdirs>=2", + "tomli>=1.1.0; python_version < \"3.11\"", + "typing-extensions>=4.0.1; python_version < \"3.11\"", +] +files = [ + {file = "black-24.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e6668650ea4b685440857138e5fe40cde4d652633b1bdffc62933d0db4ed9812"}, + {file = "black-24.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1c536fcf674217e87b8cc3657b81809d3c085d7bf3ef262ead700da345bfa6ea"}, + {file = "black-24.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:649fff99a20bd06c6f727d2a27f401331dc0cc861fb69cde910fe95b01b5928f"}, + {file = "black-24.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:fe4d6476887de70546212c99ac9bd803d90b42fc4767f058a0baa895013fbb3e"}, + {file = "black-24.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5a2221696a8224e335c28816a9d331a6c2ae15a2ee34ec857dcf3e45dbfa99ad"}, + {file = "black-24.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f9da3333530dbcecc1be13e69c250ed8dfa67f43c4005fb537bb426e19200d50"}, + {file = "black-24.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4007b1393d902b48b36958a216c20c4482f601569d19ed1df294a496eb366392"}, + {file = "black-24.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:394d4ddc64782e51153eadcaaca95144ac4c35e27ef9b0a42e121ae7e57a9175"}, + {file = "black-24.10.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b5e39e0fae001df40f95bd8cc36b9165c5e2ea88900167bddf258bacef9bbdc3"}, + {file = "black-24.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d37d422772111794b26757c5b55a3eade028aa3fde43121ab7b673d050949d65"}, + {file = "black-24.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:14b3502784f09ce2443830e3133dacf2c0110d45191ed470ecb04d0f5f6fcb0f"}, + {file = "black-24.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:30d2c30dc5139211dda799758559d1b049f7f14c580c409d6ad925b74a4208a8"}, + {file = "black-24.10.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1cbacacb19e922a1d75ef2b6ccaefcd6e93a2c05ede32f06a21386a04cedb981"}, + {file = "black-24.10.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1f93102e0c5bb3907451063e08b9876dbeac810e7da5a8bfb7aeb5a9ef89066b"}, + {file = "black-24.10.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ddacb691cdcdf77b96f549cf9591701d8db36b2f19519373d60d31746068dbf2"}, + {file = "black-24.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:680359d932801c76d2e9c9068d05c6b107f2584b2a5b88831c83962eb9984c1b"}, + {file = "black-24.10.0-py3-none-any.whl", hash = "sha256:3bb2b7a1f7b685f85b11fed1ef10f8a9148bceb49853e47a294a3dd963c1dd7d"}, + {file = "black-24.10.0.tar.gz", hash = "sha256:846ea64c97afe3bc677b761787993be4991810ecc7a4a937816dd6bddedc4875"}, +] + +[[package]] +name = "click" +version = "8.1.7" +requires_python = ">=3.7" +summary = "Composable command line interface toolkit" +groups = ["dev"] +dependencies = [ + "colorama; platform_system == \"Windows\"", + "importlib-metadata; python_version < \"3.8\"", +] +files = [ + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +summary = "Cross-platform colored terminal text." +groups = ["dev"] +marker = "sys_platform == \"win32\" or platform_system == \"Windows\"" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.2" +requires_python = ">=3.7" +summary = "Backport of PEP 654 (exception groups)" +groups = ["dev"] +marker = "python_version < \"3.11\"" +files = [ + {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"}, + {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +requires_python = ">=3.7" +summary = "brain-dead simple config-ini parsing" +groups = ["dev"] +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "magicoca" +version = "1.0.1" +requires_python = ">=3.10" +summary = "A communication library for Python" +groups = ["default"] +files = [ + {file = "magicoca-1.0.1-py3-none-any.whl", hash = "sha256:69e04be77f9c02d3d0730dc4e739246f4bdefee8b78631040b464cd98cdde51c"}, + {file = "magicoca-1.0.1.tar.gz", hash = "sha256:0dbc9a35609db92ec79076f7126566c1e71bd4b853909ecbad9221dcc7fd6f31"}, +] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +requires_python = ">=3.5" +summary = "Type system extensions for programs checked with the mypy type checker." +groups = ["dev"] +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + +[[package]] +name = "packaging" +version = "24.1" +requires_python = ">=3.8" +summary = "Core utilities for Python packages" +groups = ["dev"] +files = [ + {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, + {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, +] + +[[package]] +name = "pathspec" +version = "0.12.1" +requires_python = ">=3.8" +summary = "Utility library for gitignore style pattern matching of file paths." +groups = ["dev"] +files = [ + {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, + {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, +] + +[[package]] +name = "platformdirs" +version = "4.3.6" +requires_python = ">=3.8" +summary = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." +groups = ["dev"] +files = [ + {file = "platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"}, + {file = "platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907"}, +] + +[[package]] +name = "pluggy" +version = "1.5.0" +requires_python = ">=3.8" +summary = "plugin and hook calling mechanisms for python" +groups = ["dev"] +files = [ + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, +] + +[[package]] +name = "pytest" +version = "8.3.3" +requires_python = ">=3.8" +summary = "pytest: simple powerful testing with Python" +groups = ["dev"] +dependencies = [ + "colorama; sys_platform == \"win32\"", + "exceptiongroup>=1.0.0rc8; python_version < \"3.11\"", + "iniconfig", + "packaging", + "pluggy<2,>=1.5", + "tomli>=1; python_version < \"3.11\"", +] +files = [ + {file = "pytest-8.3.3-py3-none-any.whl", hash = "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2"}, + {file = "pytest-8.3.3.tar.gz", hash = "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181"}, +] + +[[package]] +name = "tomli" +version = "2.0.2" +requires_python = ">=3.8" +summary = "A lil' TOML parser" +groups = ["dev"] +marker = "python_version < \"3.11\"" +files = [ + {file = "tomli-2.0.2-py3-none-any.whl", hash = "sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38"}, + {file = "tomli-2.0.2.tar.gz", hash = "sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed"}, +] + +[[package]] +name = "typing-extensions" +version = "4.12.2" +requires_python = ">=3.8" +summary = "Backported and Experimental Type Hints for Python 3.8+" +groups = ["dev"] +marker = "python_version < \"3.11\"" +files = [ + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, +] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e02c880 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,30 @@ +[project] +name = "croterline" +dynamic = ["version"] +description = "Default template for PDM package" +authors = [ + {name = "snowykami", email = "snowykami@outlook.com"}, +] +dependencies = [ + "magicoca>=1.0.1", +] +requires-python = ">=3.10" +readme = "README.md" +license = {text = "MIT"} + +[build-system] +requires = ["pdm-backend"] +build-backend = "pdm.backend" + +[tool.pdm.version] +source = "scm" +tag_filter = "v*" +tag_regex = '^v(?:\D*)?(?P([1-9][0-9]*!)?(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))*((a|b|c|rc)(0|[1-9][0-9]*))?(\.post(0|[1-9][0-9]*))?(\.dev(0|[1-9][0-9]*))?$)$' + +[tool.pdm.dev-dependencies] +dev = [ + "pytest>=8.3.3", + "black>=24.10.0", +] +[tool.pdm] +distribution = true diff --git a/tests/test_subprocess.py b/tests/test_subprocess.py new file mode 100644 index 0000000..c81da83 --- /dev/null +++ b/tests/test_subprocess.py @@ -0,0 +1,34 @@ +import time + +from croterline.context import Context +from croterline.process import SubProcess, get_ctx + + +def p_func(*args, **kwargs): + i = 0 + ctx = get_ctx() + while True: + ctx.sub_chan << "Running" << "SubProcess" + print("Recv from main: ", str << ctx.main_chan) + i += 1 + if i == 5: + ctx.sub_chan << "end" + + +class TestSubProcess: + def test_run(self): + print("start") + sp = SubProcess("test", p_func, Context()) + sp.start() + + while True: + s = str << sp.ctx.sub_chan + sp.ctx.main_chan << "Resp" + print("Recv from sub: ", s) + if s == "end": + break + print("finished") + sp.terminate() + + def test_decorator(self): + pass