From 3495b750c79a6cb0134cef20bd314682c9bd061b Mon Sep 17 00:00:00 2001 From: AlexNg Date: Sun, 28 Apr 2024 17:45:24 +0800 Subject: [PATCH 01/10] refactor: Move to thread_cli --- pyproject.toml | 4 +- src/thread-cli/utils/processors.py | 41 ---------- src/{thread-cli => thread_cli}/__init__.py | 3 +- src/{thread-cli => thread_cli}/base.py | 0 src/{thread-cli => thread_cli}/process.py | 0 src/{thread-cli => thread_cli}/py.typed | 0 .../utils/__init__.py | 0 .../utils/logging.py | 0 src/thread_cli/utils/processors.py | 81 +++++++++++++++++++ 9 files changed, 85 insertions(+), 44 deletions(-) delete mode 100644 src/thread-cli/utils/processors.py rename src/{thread-cli => thread_cli}/__init__.py (91%) rename src/{thread-cli => thread_cli}/base.py (100%) rename src/{thread-cli => thread_cli}/process.py (100%) rename src/{thread-cli => thread_cli}/py.typed (100%) rename src/{thread-cli => thread_cli}/utils/__init__.py (100%) rename src/{thread-cli => thread_cli}/utils/logging.py (100%) create mode 100644 src/thread_cli/utils/processors.py diff --git a/pyproject.toml b/pyproject.toml index 92a45cb..2a1f4fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,8 +6,8 @@ authors = ["Alex "] license = "BSD-3-Clause" readme = "README.md" packages = [ - { include = "thread-cli", from = "src" }, - { include = "thread-cli/py.typed", from = "src" }, + { include = "thread_cli", from = "src" }, + { include = "thread_cli/py.typed", from = "src" }, ] include = [{ path = "tests", format = "sdist" }] homepage = "https://github.com/python-thread/thread-cli" diff --git a/src/thread-cli/utils/processors.py b/src/thread-cli/utils/processors.py deleted file mode 100644 index 75c3ab3..0000000 --- a/src/thread-cli/utils/processors.py +++ /dev/null @@ -1,41 +0,0 @@ -# Verbose Command Processor # -import typer -import logging - - -# Verbose Options # -DebugOption = typer.Option( - False, '--debug', help='Set verbosity level to [blue]DEBUG[/blue]', is_flag=True -) -VerboseOption = typer.Option( - False, - '--verbose', - '-v', - help='Set verbosity level to [green]INFO[/green]', - is_flag=True, -) -QuietOption = typer.Option( - False, '--quiet', '-q', help='Set verbosity level to [red]ERROR[/red]', is_flag=True -) - - -# Helper functions # - - -# Processors # -def verbose_args_processor(debug: bool, verbose: bool, quiet: bool): - """Handles setting and raising exceptions for verbose""" - if verbose and quiet: - raise typer.BadParameter('--quiet cannot be used with --verbose') - - if verbose and debug: - raise typer.BadParameter('--debug cannot be used with --verbose') - - logging.getLogger('base').setLevel( - ((debug and logging.DEBUG) or (verbose and logging.INFO) or logging.ERROR) - ) - - -def kwargs_processor(arguments: list[str]) -> dict[str, str]: - """Processes arguments into kwargs""" - return {kwarg[0]: kwarg[1] for i in arguments if (kwarg := i.split('='))} diff --git a/src/thread-cli/__init__.py b/src/thread_cli/__init__.py similarity index 91% rename from src/thread-cli/__init__.py rename to src/thread_cli/__init__.py index 0739168..bad8471 100644 --- a/src/thread-cli/__init__.py +++ b/src/thread_cli/__init__.py @@ -11,6 +11,7 @@ """ __version__ = '0.1.0' +from . import utils from .utils.logging import ColorLogger, logging # Export Core @@ -29,4 +30,4 @@ # Wildcard export -__all__ = ['app'] +__all__ = ['app', 'utils'] diff --git a/src/thread-cli/base.py b/src/thread_cli/base.py similarity index 100% rename from src/thread-cli/base.py rename to src/thread_cli/base.py diff --git a/src/thread-cli/process.py b/src/thread_cli/process.py similarity index 100% rename from src/thread-cli/process.py rename to src/thread_cli/process.py diff --git a/src/thread-cli/py.typed b/src/thread_cli/py.typed similarity index 100% rename from src/thread-cli/py.typed rename to src/thread_cli/py.typed diff --git a/src/thread-cli/utils/__init__.py b/src/thread_cli/utils/__init__.py similarity index 100% rename from src/thread-cli/utils/__init__.py rename to src/thread_cli/utils/__init__.py diff --git a/src/thread-cli/utils/logging.py b/src/thread_cli/utils/logging.py similarity index 100% rename from src/thread-cli/utils/logging.py rename to src/thread_cli/utils/logging.py diff --git a/src/thread_cli/utils/processors.py b/src/thread_cli/utils/processors.py new file mode 100644 index 0000000..781ee87 --- /dev/null +++ b/src/thread_cli/utils/processors.py @@ -0,0 +1,81 @@ +# Verbose Command Processor # +import os +import typer +import logging + + +# Types # +SupportedFileExtensions = ['json', ''] + + +# Verbose Options # +DebugOption = typer.Option( + False, '--debug', help='Set verbosity level to [blue]DEBUG[/blue]', is_flag=True +) +VerboseOption = typer.Option( + False, + '--verbose', + '-v', + help='Set verbosity level to [green]INFO[/green]', + is_flag=True, +) +QuietOption = typer.Option( + False, '--quiet', '-q', help='Set verbosity level to [red]ERROR[/red]', is_flag=True +) + + +# Helper functions # + + +# Processors # +def verbose_args_processor(debug: bool, verbose: bool, quiet: bool): + """Handles setting and raising exceptions for verbose""" + if verbose and quiet: + raise typer.BadParameter('--quiet cannot be used with --verbose') + + if verbose and debug: + raise typer.BadParameter('--debug cannot be used with --verbose') + + logging.getLogger('base').setLevel( + ((debug and logging.DEBUG) or (verbose and logging.INFO) or logging.ERROR) + ) + + +def kwargs_processor(arguments: list[str]) -> dict[str, str]: + """Processes arguments into kwargs""" + return { + kwarg[0]: kwarg[1] + for i in arguments + if (kwarg := i.split('=')) and (len(kwarg) == 2) + } + + +def file_processor(file: str) -> list[str]: + """Processes file path or string representation of a python data structure""" + logger = logging.getLogger('base') + logger.debug('Attempting to process as file path') + + # Handle file path + try: + if not os.path.exists(file): + raise FileNotFoundError(f'File {file} does not exist') + + if not os.path.isfile(file): + raise FileNotFoundError(f'File {file} is not a file') + + with open(file, 'r') as f: + return f.readlines() + + except Exception as e: + logger.debug('Failed to process as file path: ' + str(e)) + + # Handle string + logger.debug('Attempting to read as python data structure') + try: + dataStruct = eval(file) + return dataStruct + except Exception as e: + logger.debug('Failed to process as python data structure: ' + str(e)) + + logger.error('Failed to process file') + exit(1) From bd30b7d25358e5e9ddfc2c9ec08ba228a4f59820 Mon Sep 17 00:00:00 2001 From: AlexNg Date: Sun, 28 Apr 2024 17:46:47 +0800 Subject: [PATCH 02/10] ci: Add linter --- .github/workflows/linter.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/linter.yml diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml new file mode 100644 index 0000000..044a0b5 --- /dev/null +++ b/.github/workflows/linter.yml @@ -0,0 +1,30 @@ +name: Linting + +on: [push, pull_request] + +env: + PIP_DISABLE_PIP_VERSION_CHECK: 1 + +jobs: + linting: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python 3.x + uses: actions/setup-python@v5 + with: + python-version: 3.x + + - name: Install requirements + run: | + set -xe + python -m pip install poetry + python -m poetry install + + - name: Lint with ruff + run: | + set -xe + python -m poetry run ruff format --check From 2937b529d51500fed5029f96257ece0cd31930f1 Mon Sep 17 00:00:00 2001 From: AlexNg Date: Sun, 28 Apr 2024 17:47:02 +0800 Subject: [PATCH 03/10] ci(test): Update tests --- .github/workflows/test-worker.yml | 74 +++++++++++++++++++++---------- 1 file changed, 51 insertions(+), 23 deletions(-) diff --git a/.github/workflows/test-worker.yml b/.github/workflows/test-worker.yml index c5b8911..ea91358 100644 --- a/.github/workflows/test-worker.yml +++ b/.github/workflows/test-worker.yml @@ -1,42 +1,70 @@ -name: Run Python tests +name: Run Test Suite -on: [push] +on: + push: + pull_request: + workflow_dispatch: + +defaults: + run: + shell: bash + +env: + PIP_DISABLE_PIP_VERSION_CHECK: 1 + COVERAGE_IGOR_VERBOSE: 1 + FORCE_COLOR: 1 # pytest output color + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - build: - name: Run tests - runs-on: ubuntu-latest + tests: + name: "${{ matrix.python-version }} on ${{ matrix.os }}" + runs-on: "${{ matrix.os }}-latest" + + continue-on-error: ${{ startsWith(matrix.python-version, '~') }} # Allows unstable Python versions to fail + strategy: + fail-fast: false matrix: + os: + - ubuntu + - windows + - macos python-version: ["3.9", "3.10", "3.11", "3.x"] steps: - - uses: actions/checkout@v3 + - name: Checkout repository + uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Install Python dependencies run: | - python -m pip install -U pip - python -m pip install -U coverage pytest pytest-cov poetry + set -xe + python -m pip install poetry coverage pytest python -m poetry install - python -m poetry self add poetry-plugin-export - python -m poetry export -f requirements.txt --output requirements.txt - python -m pip install -r requirements.txt - - - name: Lint with Ruff - run: | - python -m pip install -U ruff - ruff --per-file-ignores="__init__.py:F401" --per-file-ignores="__init__.py:E402" . - continue-on-error: true - name: Test with pytest run: | - coverage run -m pytest -v -s + set -xe + python -m poetry run pytest -sv - - name: Generate Coverage Report - run: | - coverage report -m + check: + if: always() + name: Tests Successful + runs-on: ubuntu-latest + needs: tests + + steps: + - name: Whether the whole test suite passed + uses: re-actors/alls-green@v1.2.2 + with: + jobs: ${{ toJSON(needs) }} From 1810882fc1fff23f20e4df67be0ec8c9bd70110a Mon Sep 17 00:00:00 2001 From: AlexNg Date: Sun, 28 Apr 2024 17:47:11 +0800 Subject: [PATCH 04/10] test: Remove __init__ --- tests/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/__init__.py diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29..0000000 From 945029b3d8f21f2a2442b74b3ca9987d4e904581 Mon Sep 17 00:00:00 2001 From: AlexNg Date: Sun, 28 Apr 2024 17:47:23 +0800 Subject: [PATCH 05/10] test: Add testing kwargs processor --- tests/test_kwargs_processing.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 tests/test_kwargs_processing.py diff --git a/tests/test_kwargs_processing.py b/tests/test_kwargs_processing.py new file mode 100644 index 0000000..18597f8 --- /dev/null +++ b/tests/test_kwargs_processing.py @@ -0,0 +1,11 @@ +from thread_cli.utils import kwargs_processor + + +def test_normal(): + """Testing for normal arguments""" + assert kwargs_processor(['a=1', 'b=2', 'c=3', 'd=', 'e']) == { + 'a': '1', + 'b': '2', + 'c': '3', + 'd': '', + } From 5da98b6a8407939909ab3bd5776a6039386fa562 Mon Sep 17 00:00:00 2001 From: AlexNg Date: Sun, 28 Apr 2024 17:50:56 +0800 Subject: [PATCH 06/10] fix: Ignore if any side of "=" is '' --- src/thread_cli/utils/processors.py | 2 +- tests/test_kwargs_processing.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/thread_cli/utils/processors.py b/src/thread_cli/utils/processors.py index 781ee87..8c44568 100644 --- a/src/thread_cli/utils/processors.py +++ b/src/thread_cli/utils/processors.py @@ -46,7 +46,7 @@ def kwargs_processor(arguments: list[str]) -> dict[str, str]: return { kwarg[0]: kwarg[1] for i in arguments - if (kwarg := i.split('=')) and (len(kwarg) == 2) + if (kwarg := i.split('=')) and (len(kwarg) == 2) and kwarg[0] and kwarg[1] } diff --git a/tests/test_kwargs_processing.py b/tests/test_kwargs_processing.py index 18597f8..4c284f1 100644 --- a/tests/test_kwargs_processing.py +++ b/tests/test_kwargs_processing.py @@ -7,5 +7,4 @@ def test_normal(): 'a': '1', 'b': '2', 'c': '3', - 'd': '', } From 59059d8f62aed2a1f01a82769b8600860c92ba21 Mon Sep 17 00:00:00 2001 From: AlexNg Date: Sun, 28 Apr 2024 17:51:14 +0800 Subject: [PATCH 07/10] style: Ruff formatting --- src/thread_cli/base.py | 2 +- src/thread_cli/process.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/thread_cli/base.py b/src/thread_cli/base.py index 34439c5..2db6719 100644 --- a/src/thread_cli/base.py +++ b/src/thread_cli/base.py @@ -32,7 +32,7 @@ def callback( debug: bool = DebugOption, verbose: bool = VerboseOption, quiet: bool = QuietOption, - ): +): """ [b]Thread CLI[/b]\b\n [white]Use thread from the terminal![/white] diff --git a/src/thread_cli/process.py b/src/thread_cli/process.py index 344d38d..c0b1ac6 100644 --- a/src/thread_cli/process.py +++ b/src/thread_cli/process.py @@ -77,7 +77,7 @@ def process( debug: bool = DebugOption, verbose: bool = VerboseOption, quiet: bool = QuietOption, - ): +): """ [bold]Utilise parallel processing on a dataset[/bold] From 8696930a3818f893ac0ad50796c7c187fd614159 Mon Sep 17 00:00:00 2001 From: AlexNg Date: Sun, 28 Apr 2024 17:53:17 +0800 Subject: [PATCH 08/10] lint: Update ruff --- ruff.toml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ruff.toml b/ruff.toml index 12673e1..6a79578 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,5 +1,3 @@ -indent-width = 2 - [format] # Exclude commonly ignored directories. exclude = [ @@ -34,10 +32,11 @@ exclude = [ indent-style = "space" line-ending = "lf" quote-style = "single" -docstring-code-format = true [lint] +select = ["E4", "E7", "E9", "F", "B"] + # Avoid enforcing line-length violations (`E501`) ignore = ["E501"] @@ -45,6 +44,10 @@ ignore = ["E501"] dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" +[lint.flake8-quotes] +docstring-quotes = "double" + + # Ignore `E402` (import violations) in all `__init__.py` files, and in select subdirectories. [lint.per-file-ignores] "__init__.py" = ["E402", "F401"] From 90b1b04f56d33a370b52540b462ada699e8a767b Mon Sep 17 00:00:00 2001 From: AlexNg Date: Sun, 28 Apr 2024 17:53:41 +0800 Subject: [PATCH 09/10] style: Ruff formatting --- src/thread_cli/__init__.py | 6 +- src/thread_cli/base.py | 131 ++++---- src/thread_cli/process.py | 468 +++++++++++++++-------------- src/thread_cli/utils/__init__.py | 10 +- src/thread_cli/utils/logging.py | 52 ++-- src/thread_cli/utils/processors.py | 98 +++--- tests/test_kwargs_processing.py | 12 +- tests/test_placeholder.py | 4 +- 8 files changed, 394 insertions(+), 387 deletions(-) diff --git a/src/thread_cli/__init__.py b/src/thread_cli/__init__.py index bad8471..4b9255c 100644 --- a/src/thread_cli/__init__.py +++ b/src/thread_cli/__init__.py @@ -19,9 +19,9 @@ from .process import process as process_cli app.command( - name='process', - no_args_is_help=True, - context_settings={'allow_extra_args': True}, + name='process', + no_args_is_help=True, + context_settings={'allow_extra_args': True}, )(process_cli) diff --git a/src/thread_cli/base.py b/src/thread_cli/base.py index 2db6719..6846093 100644 --- a/src/thread_cli/base.py +++ b/src/thread_cli/base.py @@ -8,101 +8,104 @@ cli_base = typer.Typer( - no_args_is_help=True, - rich_markup_mode='rich', - context_settings={'help_option_names': ['-h', '--help', 'help']}, + no_args_is_help=True, + rich_markup_mode='rich', + context_settings={'help_option_names': ['-h', '--help', 'help']}, ) def version_callback(value: bool): - if value: - typer.echo(f'v{__version__}') - raise typer.Exit() + if value: + typer.echo(f'v{__version__}') + raise typer.Exit() @cli_base.callback(invoke_without_command=True) def callback( - version: bool = typer.Option( - None, - '--version', - callback=version_callback, - help='Get the current installed version', - is_eager=True, - ), - debug: bool = DebugOption, - verbose: bool = VerboseOption, - quiet: bool = QuietOption, + version: bool = typer.Option( + None, + '--version', + callback=version_callback, + help='Get the current installed version', + is_eager=True, + ), + debug: bool = DebugOption, + verbose: bool = VerboseOption, + quiet: bool = QuietOption, ): - """ - [b]Thread CLI[/b]\b\n - [white]Use thread from the terminal![/white] + """ + [b]Thread CLI[/b]\b\n + [white]Use thread from the terminal![/white] - [blue][u] [/u][/blue] + [blue][u] [/u][/blue] - Learn more from our [link=https://github.com/python-thread/thread/blob/main/docs/command-line.md]documentation![/link] - """ - verbose_args_processor(debug, verbose, quiet) + Learn more from our [link=https://github.com/python-thread/thread/blob/main/docs/command-line.md]documentation![/link] + """ + verbose_args_processor(debug, verbose, quiet) # Help and Others @cli_base.command(rich_help_panel='Help and Others') def help(): - """Get [yellow]help[/yellow] from the community. :question:""" - typer.echo('Feel free to search for or ask questions here!') - try: - logger.info('Attempting to open in web browser...') + """Get [yellow]help[/yellow] from the community. :question:""" + typer.echo('Feel free to search for or ask questions here!') + try: + logger.info('Attempting to open in web browser...') - import webbrowser + import webbrowser - webbrowser.open('https://github.com/python-thread/thread/issues', new=2) - typer.echo('Opening in web browser!') + webbrowser.open('https://github.com/python-thread/thread/issues', new=2) + typer.echo('Opening in web browser!') - except Exception as e: - logger.warn('Failed to open web browser') - logger.debug(f'{e}') - typer.echo('https://github.com/python-thread/thread/issues') + except Exception as e: + logger.warn('Failed to open web browser') + logger.debug(f'{e}') + typer.echo('https://github.com/python-thread/thread/issues') @cli_base.command(rich_help_panel='Help and Others') def docs(): - """View our [yellow]documentation.[/yellow] :book:""" - typer.echo('Thanks for using Thread, here is our documentation!') - try: - logger.info('Attempting to open in web browser...') - import webbrowser - - webbrowser.open( - 'https://github.com/python-thread/thread/blob/main/docs/command-line.md', new=2 - ) - typer.echo('Opening in web browser!') - - except Exception as e: - logger.warn('Failed to open web browser') - logger.debug(f'{e}') - typer.echo('https://github.com/python-thread/thread/blob/main/docs/command-line.md') + """View our [yellow]documentation.[/yellow] :book:""" + typer.echo('Thanks for using Thread, here is our documentation!') + try: + logger.info('Attempting to open in web browser...') + import webbrowser + + webbrowser.open( + 'https://github.com/python-thread/thread/blob/main/docs/command-line.md', + new=2, + ) + typer.echo('Opening in web browser!') + + except Exception as e: + logger.warn('Failed to open web browser') + logger.debug(f'{e}') + typer.echo( + 'https://github.com/python-thread/thread/blob/main/docs/command-line.md' + ) @cli_base.command(rich_help_panel='Help and Others') def report(): - """[yellow]Report[/yellow] an issue. :bug:""" - typer.echo('Sorry you run into an issue, report it here!') - try: - logger.info('Attempting to open in web browser...') - import webbrowser + """[yellow]Report[/yellow] an issue. :bug:""" + typer.echo('Sorry you run into an issue, report it here!') + try: + logger.info('Attempting to open in web browser...') + import webbrowser - webbrowser.open('https://github.com/python-thread/thread/issues', new=2) - typer.echo('Opening in web browser!') + webbrowser.open('https://github.com/python-thread/thread/issues', new=2) + typer.echo('Opening in web browser!') - except Exception as e: - logger.warn('Failed to open web browser') - logger.debug(f'{e}') - typer.echo('https://github.com/python-thread/thread/issues') + except Exception as e: + logger.warn('Failed to open web browser') + logger.debug(f'{e}') + typer.echo('https://github.com/python-thread/thread/issues') # Utils and Configs @cli_base.command(rich_help_panel='Utils and Configs') def config(configuration: str): - """ - [blue]Configure[/blue] the system. :wrench: - """ - typer.echo('Coming soon!') + """ + [blue]Configure[/blue] the system. :wrench: + """ + typer.echo('Coming soon!') diff --git a/src/thread_cli/process.py b/src/thread_cli/process.py index c0b1ac6..1f70971 100644 --- a/src/thread_cli/process.py +++ b/src/thread_cli/process.py @@ -14,250 +14,254 @@ from rich.panel import Panel from rich.console import Group from rich.progress import ( - Progress, - TaskID, - SpinnerColumn, - TextColumn, - BarColumn, - TimeRemainingColumn, - TimeElapsedColumn, + Progress, + TaskID, + SpinnerColumn, + TextColumn, + BarColumn, + TimeRemainingColumn, + TimeElapsedColumn, ) from .utils import ( - DebugOption, - VerboseOption, - QuietOption, - verbose_args_processor, - kwargs_processor, + DebugOption, + VerboseOption, + QuietOption, + verbose_args_processor, + kwargs_processor, ) logger = logging.getLogger('base') def process( - func: str = typer.Argument( - help='[blue].path.to.file[/blue]:[blue]function_name[/blue] OR [blue]lambda x: x[/blue]' - ), - dataset: str = typer.Argument( - help='[blue]./path/to/file.txt[/blue] OR [blue][ i for i in range(2) ][/blue]' - ), - args: list[str] = typer.Option( - [], '--arg', '-a', help='[blue]Arguments[/blue] passed to each thread' - ), - kargs: list[str] = typer.Option( - [], '--kwarg', '-kw', help='[blue]Key-Value arguments[/blue] passed to each thread' - ), - threads: int = typer.Option( - 8, - '--threads', - '-t', - help='Maximum number of [blue]threads[/blue] (will scale down based on dataset size)', - ), - daemon: bool = typer.Option( - False, '--daemon', '-d', help='Threads to run in [blue]daemon[/blue] mode' - ), - graceful_exit: bool = typer.Option( - True, - '--graceful-exit', - '-ge', - is_flag=True, - help='Whether to [blue]gracefully exit[/blue] on abrupt exit (etc. CTRL+C)', - ), - output: str = typer.Option( - './output.json', '--output', '-o', help='[blue]Output[/blue] file location' - ), - fileout: bool = typer.Option( - True, - '--fileout', - is_flag=True, - help='Whether to [blue]write[/blue] output to a file', - ), - stdout: bool = typer.Option( - False, '--stdout', is_flag=True, help='Whether to [blue]print[/blue] the output' - ), - debug: bool = DebugOption, - verbose: bool = VerboseOption, - quiet: bool = QuietOption, + func: str = typer.Argument( + help='[blue].path.to.file[/blue]:[blue]function_name[/blue] OR [blue]lambda x: x[/blue]' + ), + dataset: str = typer.Argument( + help='[blue]./path/to/file.txt[/blue] OR [blue][ i for i in range(2) ][/blue]' + ), + args: list[str] = typer.Option( + [], '--arg', '-a', help='[blue]Arguments[/blue] passed to each thread' + ), + kargs: list[str] = typer.Option( + [], + '--kwarg', + '-kw', + help='[blue]Key-Value arguments[/blue] passed to each thread', + ), + threads: int = typer.Option( + 8, + '--threads', + '-t', + help='Maximum number of [blue]threads[/blue] (will scale down based on dataset size)', + ), + daemon: bool = typer.Option( + False, '--daemon', '-d', help='Threads to run in [blue]daemon[/blue] mode' + ), + graceful_exit: bool = typer.Option( + True, + '--graceful-exit', + '-ge', + is_flag=True, + help='Whether to [blue]gracefully exit[/blue] on abrupt exit (etc. CTRL+C)', + ), + output: str = typer.Option( + './output.json', '--output', '-o', help='[blue]Output[/blue] file location' + ), + fileout: bool = typer.Option( + True, + '--fileout', + is_flag=True, + help='Whether to [blue]write[/blue] output to a file', + ), + stdout: bool = typer.Option( + False, '--stdout', is_flag=True, help='Whether to [blue]print[/blue] the output' + ), + debug: bool = DebugOption, + verbose: bool = VerboseOption, + quiet: bool = QuietOption, ): - """ - [bold]Utilise parallel processing on a dataset[/bold] - - \b\n - [bold white]:glowing_star: Important[/bold white] - Args and Kwargs can be parsed by adding multiple -a or -kw - - [green]$ thread[/green] [blue]process[/blue] ... -a 'an arg' -kw myKey=myValue -arg testing --kwarg a1=a2 - [white]=> args = [ [green]'an arg'[/green], [green]'testing'[/green] ][/white] - [white] kwargs = { [green]'myKey'[/green]: [green]'myValue'[/green], [green]'a1'[/green]: [green]'a2'[/green] }[/white] - - [blue][u] [/u][/blue] - - Learn more from our [link=https://github.com/python-thread/thread/blob/main/docs/command-line.md#parallel-processing-thread-process]documentation![/link] - """ - verbose_args_processor(debug, verbose, quiet) - kwargs = kwargs_processor(kargs) - logger.debug('Processed kwargs: %s' % kwargs) - - # Verify output - if not fileout and not stdout: - raise typer.BadParameter('No output method specified') - - if fileout and not os.path.exists('/'.join(output.split('/')[:-1])): - raise typer.BadParameter('Output file directory does not exist') - - # Loading function - f = None - try: - logger.info('Attempted to interpret function') - f = eval( - func - ) # I know eval is bad practice, but I have yet to find a safer replacement - logger.debug('Evaluated function: %s' % f) - - if not inspect.isfunction(f): - logger.info('Invalid function') - except Exception: - logger.info('Failed to interpret function') - - if not f: + """ + [bold]Utilise parallel processing on a dataset[/bold] + + \b\n + [bold white]:glowing_star: Important[/bold white] + Args and Kwargs can be parsed by adding multiple -a or -kw + + [green]$ thread[/green] [blue]process[/blue] ... -a 'an arg' -kw myKey=myValue -arg testing --kwarg a1=a2 + [white]=> args = [ [green]'an arg'[/green], [green]'testing'[/green] ][/white] + [white] kwargs = { [green]'myKey'[/green]: [green]'myValue'[/green], [green]'a1'[/green]: [green]'a2'[/green] }[/white] + + [blue][u] [/u][/blue] + + Learn more from our [link=https://github.com/python-thread/thread/blob/main/docs/command-line.md#parallel-processing-thread-process]documentation![/link] + """ + verbose_args_processor(debug, verbose, quiet) + kwargs = kwargs_processor(kargs) + logger.debug('Processed kwargs: %s' % kwargs) + + # Verify output + if not fileout and not stdout: + raise typer.BadParameter('No output method specified') + + if fileout and not os.path.exists('/'.join(output.split('/')[:-1])): + raise typer.BadParameter('Output file directory does not exist') + + # Loading function + f = None + try: + logger.info('Attempted to interpret function') + f = eval( + func + ) # I know eval is bad practice, but I have yet to find a safer replacement + logger.debug('Evaluated function: %s' % f) + + if not inspect.isfunction(f): + logger.info('Invalid function') + except Exception: + logger.info('Failed to interpret function') + + if not f: + try: + logger.info('Attempting to fetch function file') + + fPath, fName = func.split(':') + f = importlib.import_module(fPath).__dict__[fName] + logger.debug('Evaluated function: %s' % f) + + if not inspect.isfunction(f): + logger.info('Not a function') + raise Exception('Not a function') + except Exception as e: + logger.warning('Failed to fetch function') + raise typer.BadParameter('Failed to fetch function') from e + + # Loading dataset + ds: Union[list, tuple, set, None] = None try: - logger.info('Attempting to fetch function file') - - fPath, fName = func.split(':') - f = importlib.import_module(fPath).__dict__[fName] - logger.debug('Evaluated function: %s' % f) - - if not inspect.isfunction(f): - logger.info('Not a function') - raise Exception('Not a function') - except Exception as e: - logger.warning('Failed to fetch function') - raise typer.BadParameter('Failed to fetch function') from e - - # Loading dataset - ds: Union[list, tuple, set, None] = None - try: - logger.info('Attempting to interpret dataset') - ds = eval(dataset) - logger.debug( - 'Evaluated dataset: %s' % (str(ds)[:125] + '...' if len(str(ds)) > 125 else ds) + logger.info('Attempting to interpret dataset') + ds = eval(dataset) + logger.debug( + 'Evaluated dataset: %s' + % (str(ds)[:125] + '...' if len(str(ds)) > 125 else ds) + ) + + if not isinstance(ds, (list, tuple, set)): + logger.info('Invalid dataset literal') + ds = None + + except Exception: + logger.info('Failed to interpret dataset') + + if not ds: + try: + logger.info('Attempting to fetch data file') + if not os.path.isfile(dataset): + logger.info('Invalid file path') + raise Exception('Invalid file path') + + with open(dataset, 'r') as a: + ds = [i.endswith('\n') and i[:-2] for i in a.readlines()] + except Exception as e: + logger.warning('Failed to read dataset') + raise typer.BadParameter('Failed to read dataset') from e + + logger.info('Interpreted dataset') + + # Setup + logger.debug('Importing module') + from thread import Settings, ParallelProcessing + + logger.info( + 'Spawning threads... [Expected: {tcount} threads]'.format( + tcount=min(len(ds), threads) + ) ) - if not isinstance(ds, (list, tuple, set)): - logger.info('Invalid dataset literal') - ds = None + Settings.set_graceful_exit(graceful_exit) + newProcess = ParallelProcessing( + function=f, + dataset=list(ds), + args=args, + kwargs=kwargs, + daemon=daemon, + max_threads=threads, + ) - except Exception: - logger.info('Failed to interpret dataset') + logger.info('Created parallel process') + logger.info('Starting parallel process') - if not ds: - try: - logger.info('Attempting to fetch data file') - if not os.path.isfile(dataset): - logger.info('Invalid file path') - raise Exception('Invalid file path') - - with open(dataset, 'r') as a: - ds = [i.endswith('\n') and i[:-2] for i in a.readlines()] - except Exception as e: - logger.warning('Failed to read dataset') - raise typer.BadParameter('Failed to read dataset') from e - - logger.info('Interpreted dataset') - - # Setup - logger.debug('Importing module') - from thread import Settings, ParallelProcessing - - logger.info( - 'Spawning threads... [Expected: {tcount} threads]'.format( - tcount=min(len(ds), threads) + start_t = time.perf_counter() + newProcess.start() + + logger.info('Started parallel processes') + typer.echo('Waiting for parallel processes to complete, this may take a while...') + + # Progress bar :D + threadCount = len(newProcess._threads) + + thread_progress = Progress( + SpinnerColumn(), + TextColumn('{task.description}'), + '•', + TimeRemainingColumn(), + BarColumn(bar_width=80), + TextColumn('{task.percentage:>3.1f}%'), ) - ) - - Settings.set_graceful_exit(graceful_exit) - newProcess = ParallelProcessing( - function=f, - dataset=list(ds), - args=args, - kwargs=kwargs, - daemon=daemon, - max_threads=threads, - ) - - logger.info('Created parallel process') - logger.info('Starting parallel process') - - start_t = time.perf_counter() - newProcess.start() - - logger.info('Started parallel processes') - typer.echo('Waiting for parallel processes to complete, this may take a while...') - - # Progress bar :D - threadCount = len(newProcess._threads) - - thread_progress = Progress( - SpinnerColumn(), - TextColumn('{task.description}'), - '•', - TimeRemainingColumn(), - BarColumn(bar_width=80), - TextColumn('{task.percentage:>3.1f}%'), - ) - overall_progress = Progress( - TimeElapsedColumn(), BarColumn(bar_width=110), TextColumn('{task.description}') - ) - - workerjobs: list[TaskID] = [ - thread_progress.add_task(f'[bold blue][T {threadNum}]', total=100) - for threadNum in range(threadCount) - ] - overalljob = overall_progress.add_task('(0 of ?)', total=100) - - with Live( - Group( - Panel(thread_progress), - overall_progress, - ), - refresh_per_second=10, - ): - completed = 0 - while completed != threadCount: - i = 0 - completed = 0 - progressAvg = 0 - - for jobID in workerjobs: - jobProgress = newProcess._threads[i].progress - thread_progress.update(jobID, completed=round(jobProgress * 100, 2)) - if jobProgress == 1: - thread_progress.stop_task(jobID) - thread_progress.update(jobID, description='[bold green]Completed') - completed += 1 - - progressAvg += jobProgress - i += 1 - - # Update overall - overall_progress.update( - overalljob, - description=f'[bold {"green" if completed == threadCount else "#AAAAAA"}]({completed} of {threadCount})', - completed=round((progressAvg / threadCount) * 100, 2), - ) - time.sleep(0.1) - - result = newProcess.get_return_values() - - typer.echo(f'Completed in {(time.perf_counter() - start_t):.5f}s') - if fileout: - typer.echo(f'Writing to {output}') - try: - with open(output, 'w') as f: - json.dump(result, f, indent=2) - logger.info('Wrote to file') - except Exception as e: - logger.error('Failed to write to file') - logger.debug(str(e)) - - if stdout: - typer.echo(result) + overall_progress = Progress( + TimeElapsedColumn(), BarColumn(bar_width=110), TextColumn('{task.description}') + ) + + workerjobs: list[TaskID] = [ + thread_progress.add_task(f'[bold blue][T {threadNum}]', total=100) + for threadNum in range(threadCount) + ] + overalljob = overall_progress.add_task('(0 of ?)', total=100) + + with Live( + Group( + Panel(thread_progress), + overall_progress, + ), + refresh_per_second=10, + ): + completed = 0 + while completed != threadCount: + i = 0 + completed = 0 + progressAvg = 0 + + for jobID in workerjobs: + jobProgress = newProcess._threads[i].progress + thread_progress.update(jobID, completed=round(jobProgress * 100, 2)) + if jobProgress == 1: + thread_progress.stop_task(jobID) + thread_progress.update(jobID, description='[bold green]Completed') + completed += 1 + + progressAvg += jobProgress + i += 1 + + # Update overall + overall_progress.update( + overalljob, + description=f'[bold {"green" if completed == threadCount else "#AAAAAA"}]({completed} of {threadCount})', + completed=round((progressAvg / threadCount) * 100, 2), + ) + time.sleep(0.1) + + result = newProcess.get_return_values() + + typer.echo(f'Completed in {(time.perf_counter() - start_t):.5f}s') + if fileout: + typer.echo(f'Writing to {output}') + try: + with open(output, 'w') as f: + json.dump(result, f, indent=2) + logger.info('Wrote to file') + except Exception as e: + logger.error('Failed to write to file') + logger.debug(str(e)) + + if stdout: + typer.echo(result) diff --git a/src/thread_cli/utils/__init__.py b/src/thread_cli/utils/__init__.py index 30edbd9..1ffcd5b 100644 --- a/src/thread_cli/utils/__init__.py +++ b/src/thread_cli/utils/__init__.py @@ -1,8 +1,8 @@ from . import logging from .processors import ( - verbose_args_processor, - kwargs_processor, - DebugOption, - VerboseOption, - QuietOption, + verbose_args_processor, + kwargs_processor, + DebugOption, + VerboseOption, + QuietOption, ) diff --git a/src/thread_cli/utils/logging.py b/src/thread_cli/utils/logging.py index 1d598a8..fe58999 100644 --- a/src/thread_cli/utils/logging.py +++ b/src/thread_cli/utils/logging.py @@ -6,34 +6,34 @@ # Stdout color configuration class ColorFormatter(logging.Formatter): - COLORS = { - 'DEBUG': Fore.BLUE, - 'INFO': Fore.GREEN, - 'WARNING': Fore.YELLOW, - 'ERROR': Fore.RED, - 'CRITICAL': Fore.RED + Style.BRIGHT, - } - - def format(self, record): - color = self.COLORS.get(record.levelname, '') - record.levelname = ( - record.levelname - if not color - else color + Style.BRIGHT + f'{record.levelname:<9}|' - ) - record.msg = ( - record.msg if not color else color + Fore.WHITE + Style.NORMAL + record.msg - ) - - return logging.Formatter.format(self, record) + COLORS = { + 'DEBUG': Fore.BLUE, + 'INFO': Fore.GREEN, + 'WARNING': Fore.YELLOW, + 'ERROR': Fore.RED, + 'CRITICAL': Fore.RED + Style.BRIGHT, + } + + def format(self, record): + color = self.COLORS.get(record.levelname, '') + record.levelname = ( + record.levelname + if not color + else color + Style.BRIGHT + f'{record.levelname:<9}|' + ) + record.msg = ( + record.msg if not color else color + Fore.WHITE + Style.NORMAL + record.msg + ) + + return logging.Formatter.format(self, record) # Configure logger class ColorLogger(logging.Logger): - def __init__(self, name): - logging.Logger.__init__(self, name, logging.DEBUG) - colorFormatter = ColorFormatter('%(levelname)s %(message)s' + Style.RESET_ALL) + def __init__(self, name): + logging.Logger.__init__(self, name, logging.DEBUG) + colorFormatter = ColorFormatter('%(levelname)s %(message)s' + Style.RESET_ALL) - console = logging.StreamHandler() - console.setFormatter(colorFormatter) - self.addHandler(console) + console = logging.StreamHandler() + console.setFormatter(colorFormatter) + self.addHandler(console) diff --git a/src/thread_cli/utils/processors.py b/src/thread_cli/utils/processors.py index 8c44568..0d9b0a9 100644 --- a/src/thread_cli/utils/processors.py +++ b/src/thread_cli/utils/processors.py @@ -10,17 +10,17 @@ # Verbose Options # DebugOption = typer.Option( - False, '--debug', help='Set verbosity level to [blue]DEBUG[/blue]', is_flag=True + False, '--debug', help='Set verbosity level to [blue]DEBUG[/blue]', is_flag=True ) VerboseOption = typer.Option( - False, - '--verbose', - '-v', - help='Set verbosity level to [green]INFO[/green]', - is_flag=True, + False, + '--verbose', + '-v', + help='Set verbosity level to [green]INFO[/green]', + is_flag=True, ) QuietOption = typer.Option( - False, '--quiet', '-q', help='Set verbosity level to [red]ERROR[/red]', is_flag=True + False, '--quiet', '-q', help='Set verbosity level to [red]ERROR[/red]', is_flag=True ) @@ -29,53 +29,53 @@ # Processors # def verbose_args_processor(debug: bool, verbose: bool, quiet: bool): - """Handles setting and raising exceptions for verbose""" - if verbose and quiet: - raise typer.BadParameter('--quiet cannot be used with --verbose') + """Handles setting and raising exceptions for verbose""" + if verbose and quiet: + raise typer.BadParameter('--quiet cannot be used with --verbose') - if verbose and debug: - raise typer.BadParameter('--debug cannot be used with --verbose') + if verbose and debug: + raise typer.BadParameter('--debug cannot be used with --verbose') - logging.getLogger('base').setLevel( - ((debug and logging.DEBUG) or (verbose and logging.INFO) or logging.ERROR) - ) + logging.getLogger('base').setLevel( + ((debug and logging.DEBUG) or (verbose and logging.INFO) or logging.ERROR) + ) def kwargs_processor(arguments: list[str]) -> dict[str, str]: - """Processes arguments into kwargs""" - return { - kwarg[0]: kwarg[1] - for i in arguments - if (kwarg := i.split('=')) and (len(kwarg) == 2) and kwarg[0] and kwarg[1] - } + """Processes arguments into kwargs""" + return { + kwarg[0]: kwarg[1] + for i in arguments + if (kwarg := i.split('=')) and (len(kwarg) == 2) and kwarg[0] and kwarg[1] + } def file_processor(file: str) -> list[str]: - """Processes file path or string representation of a python data structure""" - logger = logging.getLogger('base') - logger.debug('Attempting to process as file path') - - # Handle file path - try: - if not os.path.exists(file): - raise FileNotFoundError(f'File {file} does not exist') - - if not os.path.isfile(file): - raise FileNotFoundError(f'File {file} is not a file') - - with open(file, 'r') as f: - return f.readlines() - - except Exception as e: - logger.debug('Failed to process as file path: ' + str(e)) - - # Handle string - logger.debug('Attempting to read as python data structure') - try: - dataStruct = eval(file) - return dataStruct - except Exception as e: - logger.debug('Failed to process as python data structure: ' + str(e)) - - logger.error('Failed to process file') - exit(1) + """Processes file path or string representation of a python data structure""" + logger = logging.getLogger('base') + logger.debug('Attempting to process as file path') + + # Handle file path + try: + if not os.path.exists(file): + raise FileNotFoundError(f'File {file} does not exist') + + if not os.path.isfile(file): + raise FileNotFoundError(f'File {file} is not a file') + + with open(file, 'r') as f: + return f.readlines() + + except Exception as e: + logger.debug('Failed to process as file path: ' + str(e)) + + # Handle string + logger.debug('Attempting to read as python data structure') + try: + dataStruct = eval(file) + return dataStruct + except Exception as e: + logger.debug('Failed to process as python data structure: ' + str(e)) + + logger.error('Failed to process file') + exit(1) diff --git a/tests/test_kwargs_processing.py b/tests/test_kwargs_processing.py index 4c284f1..ab924fa 100644 --- a/tests/test_kwargs_processing.py +++ b/tests/test_kwargs_processing.py @@ -2,9 +2,9 @@ def test_normal(): - """Testing for normal arguments""" - assert kwargs_processor(['a=1', 'b=2', 'c=3', 'd=', 'e']) == { - 'a': '1', - 'b': '2', - 'c': '3', - } + """Testing for normal arguments""" + assert kwargs_processor(['a=1', 'b=2', 'c=3', 'd=', 'e']) == { + 'a': '1', + 'b': '2', + 'c': '3', + } diff --git a/tests/test_placeholder.py b/tests/test_placeholder.py index d20e63f..aea6d5f 100644 --- a/tests/test_placeholder.py +++ b/tests/test_placeholder.py @@ -1,4 +1,4 @@ def test_packagesExist(): - import thread + import thread - assert thread + assert thread From df3c7b8d49bc73a6cefb60f3f84d994767849bd9 Mon Sep 17 00:00:00 2001 From: AlexNg Date: Sun, 28 Apr 2024 18:06:34 +0800 Subject: [PATCH 10/10] chor(license): Fix license header --- src/thread_cli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/thread_cli/__init__.py b/src/thread_cli/__init__.py index 9dae36e..caf8cbb 100644 --- a/src/thread_cli/__init__.py +++ b/src/thread_cli/__init__.py @@ -5,7 +5,7 @@ --- -Released under the GPG-3 License +Released under the BSD-3 License Copyright (c) 2020, thread.ngjx.org. All rights reserved. """