diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml new file mode 100644 index 0000000..044a0b5 --- /dev/null +++ b/.github/workflows/linter.yml @@ -0,0 +1,30 @@ +name: Linting + +on: [push, pull_request] + +env: + PIP_DISABLE_PIP_VERSION_CHECK: 1 + +jobs: + linting: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python 3.x + uses: actions/setup-python@v5 + with: + python-version: 3.x + + - name: Install requirements + run: | + set -xe + python -m pip install poetry + python -m poetry install + + - name: Lint with ruff + run: | + set -xe + python -m poetry run ruff format --check diff --git a/.github/workflows/test-worker.yml b/.github/workflows/test-worker.yml index c5b8911..ea91358 100644 --- a/.github/workflows/test-worker.yml +++ b/.github/workflows/test-worker.yml @@ -1,42 +1,70 @@ -name: Run Python tests +name: Run Test Suite -on: [push] +on: + push: + pull_request: + workflow_dispatch: + +defaults: + run: + shell: bash + +env: + PIP_DISABLE_PIP_VERSION_CHECK: 1 + COVERAGE_IGOR_VERBOSE: 1 + FORCE_COLOR: 1 # pytest output color + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - build: - name: Run tests - runs-on: ubuntu-latest + tests: + name: "${{ matrix.python-version }} on ${{ matrix.os }}" + runs-on: "${{ matrix.os }}-latest" + + continue-on-error: ${{ startsWith(matrix.python-version, '~') }} # Allows unstable Python versions to fail + strategy: + fail-fast: false matrix: + os: + - ubuntu + - windows + - macos python-version: ["3.9", "3.10", "3.11", "3.x"] steps: - - uses: actions/checkout@v3 + - name: Checkout repository + uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Install Python dependencies run: | - python -m pip install -U pip - python -m pip install -U coverage pytest pytest-cov poetry + set -xe + python -m pip install poetry coverage pytest python -m poetry install - python -m poetry self add poetry-plugin-export - python -m poetry export -f requirements.txt --output requirements.txt - python -m pip install -r requirements.txt - - - name: Lint with Ruff - run: | - python -m pip install -U ruff - ruff --per-file-ignores="__init__.py:F401" --per-file-ignores="__init__.py:E402" . - continue-on-error: true - name: Test with pytest run: | - coverage run -m pytest -v -s + set -xe + python -m poetry run pytest -sv - - name: Generate Coverage Report - run: | - coverage report -m + check: + if: always() + name: Tests Successful + runs-on: ubuntu-latest + needs: tests + + steps: + - name: Whether the whole test suite passed + uses: re-actors/alls-green@v1.2.2 + with: + jobs: ${{ toJSON(needs) }} diff --git a/pyproject.toml b/pyproject.toml index 3f851ed..4e09cbb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,8 +6,8 @@ authors = ["Alex "] license = "BSD-3-Clause" readme = "README.md" packages = [ - { include = "thread-cli", from = "src" }, - { include = "thread-cli/py.typed", from = "src" }, + { include = "thread_cli", from = "src" }, + { include = "thread_cli/py.typed", from = "src" }, ] include = [{ path = "tests", format = "sdist" }] homepage = "https://github.com/python-thread/thread-cli" diff --git a/ruff.toml b/ruff.toml index 12673e1..6a79578 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,5 +1,3 @@ -indent-width = 2 - [format] # Exclude commonly ignored directories. exclude = [ @@ -34,10 +32,11 @@ exclude = [ indent-style = "space" line-ending = "lf" quote-style = "single" -docstring-code-format = true [lint] +select = ["E4", "E7", "E9", "F", "B"] + # Avoid enforcing line-length violations (`E501`) ignore = ["E501"] @@ -45,6 +44,10 @@ ignore = ["E501"] dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" +[lint.flake8-quotes] +docstring-quotes = "double" + + # Ignore `E402` (import violations) in all `__init__.py` files, and in select subdirectories. [lint.per-file-ignores] "__init__.py" = ["E402", "F401"] diff --git a/src/thread-cli/base.py b/src/thread-cli/base.py deleted file mode 100644 index 34439c5..0000000 --- a/src/thread-cli/base.py +++ /dev/null @@ -1,108 +0,0 @@ -import typer -import logging - -from . import __version__ -from .utils import DebugOption, VerboseOption, QuietOption, verbose_args_processor - -logger = logging.getLogger('base') - - -cli_base = typer.Typer( - no_args_is_help=True, - rich_markup_mode='rich', - context_settings={'help_option_names': ['-h', '--help', 'help']}, -) - - -def version_callback(value: bool): - if value: - typer.echo(f'v{__version__}') - raise typer.Exit() - - -@cli_base.callback(invoke_without_command=True) -def callback( - version: bool = typer.Option( - None, - '--version', - callback=version_callback, - help='Get the current installed version', - is_eager=True, - ), - debug: bool = DebugOption, - verbose: bool = VerboseOption, - quiet: bool = QuietOption, - ): - """ - [b]Thread CLI[/b]\b\n - [white]Use thread from the terminal![/white] - - [blue][u] [/u][/blue] - - Learn more from our [link=https://github.com/python-thread/thread/blob/main/docs/command-line.md]documentation![/link] - """ - verbose_args_processor(debug, verbose, quiet) - - -# Help and Others -@cli_base.command(rich_help_panel='Help and Others') -def help(): - """Get [yellow]help[/yellow] from the community. :question:""" - typer.echo('Feel free to search for or ask questions here!') - try: - logger.info('Attempting to open in web browser...') - - import webbrowser - - webbrowser.open('https://github.com/python-thread/thread/issues', new=2) - typer.echo('Opening in web browser!') - - except Exception as e: - logger.warn('Failed to open web browser') - logger.debug(f'{e}') - typer.echo('https://github.com/python-thread/thread/issues') - - -@cli_base.command(rich_help_panel='Help and Others') -def docs(): - """View our [yellow]documentation.[/yellow] :book:""" - typer.echo('Thanks for using Thread, here is our documentation!') - try: - logger.info('Attempting to open in web browser...') - import webbrowser - - webbrowser.open( - 'https://github.com/python-thread/thread/blob/main/docs/command-line.md', new=2 - ) - typer.echo('Opening in web browser!') - - except Exception as e: - logger.warn('Failed to open web browser') - logger.debug(f'{e}') - typer.echo('https://github.com/python-thread/thread/blob/main/docs/command-line.md') - - -@cli_base.command(rich_help_panel='Help and Others') -def report(): - """[yellow]Report[/yellow] an issue. :bug:""" - typer.echo('Sorry you run into an issue, report it here!') - try: - logger.info('Attempting to open in web browser...') - import webbrowser - - webbrowser.open('https://github.com/python-thread/thread/issues', new=2) - typer.echo('Opening in web browser!') - - except Exception as e: - logger.warn('Failed to open web browser') - logger.debug(f'{e}') - typer.echo('https://github.com/python-thread/thread/issues') - - -# Utils and Configs -@cli_base.command(rich_help_panel='Utils and Configs') -def config(configuration: str): - """ - [blue]Configure[/blue] the system. :wrench: - """ - typer.echo('Coming soon!') diff --git a/src/thread-cli/process.py b/src/thread-cli/process.py deleted file mode 100644 index 344d38d..0000000 --- a/src/thread-cli/process.py +++ /dev/null @@ -1,263 +0,0 @@ -"""Parallel Processing command""" - -import os -import time -import json -import inspect -import importlib - -import typer -import logging -from typing import Union - -from rich.live import Live -from rich.panel import Panel -from rich.console import Group -from rich.progress import ( - Progress, - TaskID, - SpinnerColumn, - TextColumn, - BarColumn, - TimeRemainingColumn, - TimeElapsedColumn, -) -from .utils import ( - DebugOption, - VerboseOption, - QuietOption, - verbose_args_processor, - kwargs_processor, -) - -logger = logging.getLogger('base') - - -def process( - func: str = typer.Argument( - help='[blue].path.to.file[/blue]:[blue]function_name[/blue] OR [blue]lambda x: x[/blue]' - ), - dataset: str = typer.Argument( - help='[blue]./path/to/file.txt[/blue] OR [blue][ i for i in range(2) ][/blue]' - ), - args: list[str] = typer.Option( - [], '--arg', '-a', help='[blue]Arguments[/blue] passed to each thread' - ), - kargs: list[str] = typer.Option( - [], '--kwarg', '-kw', help='[blue]Key-Value arguments[/blue] passed to each thread' - ), - threads: int = typer.Option( - 8, - '--threads', - '-t', - help='Maximum number of [blue]threads[/blue] (will scale down based on dataset size)', - ), - daemon: bool = typer.Option( - False, '--daemon', '-d', help='Threads to run in [blue]daemon[/blue] mode' - ), - graceful_exit: bool = typer.Option( - True, - '--graceful-exit', - '-ge', - is_flag=True, - help='Whether to [blue]gracefully exit[/blue] on abrupt exit (etc. CTRL+C)', - ), - output: str = typer.Option( - './output.json', '--output', '-o', help='[blue]Output[/blue] file location' - ), - fileout: bool = typer.Option( - True, - '--fileout', - is_flag=True, - help='Whether to [blue]write[/blue] output to a file', - ), - stdout: bool = typer.Option( - False, '--stdout', is_flag=True, help='Whether to [blue]print[/blue] the output' - ), - debug: bool = DebugOption, - verbose: bool = VerboseOption, - quiet: bool = QuietOption, - ): - """ - [bold]Utilise parallel processing on a dataset[/bold] - - \b\n - [bold white]:glowing_star: Important[/bold white] - Args and Kwargs can be parsed by adding multiple -a or -kw - - [green]$ thread[/green] [blue]process[/blue] ... -a 'an arg' -kw myKey=myValue -arg testing --kwarg a1=a2 - [white]=> args = [ [green]'an arg'[/green], [green]'testing'[/green] ][/white] - [white] kwargs = { [green]'myKey'[/green]: [green]'myValue'[/green], [green]'a1'[/green]: [green]'a2'[/green] }[/white] - - [blue][u] [/u][/blue] - - Learn more from our [link=https://github.com/python-thread/thread/blob/main/docs/command-line.md#parallel-processing-thread-process]documentation![/link] - """ - verbose_args_processor(debug, verbose, quiet) - kwargs = kwargs_processor(kargs) - logger.debug('Processed kwargs: %s' % kwargs) - - # Verify output - if not fileout and not stdout: - raise typer.BadParameter('No output method specified') - - if fileout and not os.path.exists('/'.join(output.split('/')[:-1])): - raise typer.BadParameter('Output file directory does not exist') - - # Loading function - f = None - try: - logger.info('Attempted to interpret function') - f = eval( - func - ) # I know eval is bad practice, but I have yet to find a safer replacement - logger.debug('Evaluated function: %s' % f) - - if not inspect.isfunction(f): - logger.info('Invalid function') - except Exception: - logger.info('Failed to interpret function') - - if not f: - try: - logger.info('Attempting to fetch function file') - - fPath, fName = func.split(':') - f = importlib.import_module(fPath).__dict__[fName] - logger.debug('Evaluated function: %s' % f) - - if not inspect.isfunction(f): - logger.info('Not a function') - raise Exception('Not a function') - except Exception as e: - logger.warning('Failed to fetch function') - raise typer.BadParameter('Failed to fetch function') from e - - # Loading dataset - ds: Union[list, tuple, set, None] = None - try: - logger.info('Attempting to interpret dataset') - ds = eval(dataset) - logger.debug( - 'Evaluated dataset: %s' % (str(ds)[:125] + '...' if len(str(ds)) > 125 else ds) - ) - - if not isinstance(ds, (list, tuple, set)): - logger.info('Invalid dataset literal') - ds = None - - except Exception: - logger.info('Failed to interpret dataset') - - if not ds: - try: - logger.info('Attempting to fetch data file') - if not os.path.isfile(dataset): - logger.info('Invalid file path') - raise Exception('Invalid file path') - - with open(dataset, 'r') as a: - ds = [i.endswith('\n') and i[:-2] for i in a.readlines()] - except Exception as e: - logger.warning('Failed to read dataset') - raise typer.BadParameter('Failed to read dataset') from e - - logger.info('Interpreted dataset') - - # Setup - logger.debug('Importing module') - from thread import Settings, ParallelProcessing - - logger.info( - 'Spawning threads... [Expected: {tcount} threads]'.format( - tcount=min(len(ds), threads) - ) - ) - - Settings.set_graceful_exit(graceful_exit) - newProcess = ParallelProcessing( - function=f, - dataset=list(ds), - args=args, - kwargs=kwargs, - daemon=daemon, - max_threads=threads, - ) - - logger.info('Created parallel process') - logger.info('Starting parallel process') - - start_t = time.perf_counter() - newProcess.start() - - logger.info('Started parallel processes') - typer.echo('Waiting for parallel processes to complete, this may take a while...') - - # Progress bar :D - threadCount = len(newProcess._threads) - - thread_progress = Progress( - SpinnerColumn(), - TextColumn('{task.description}'), - '•', - TimeRemainingColumn(), - BarColumn(bar_width=80), - TextColumn('{task.percentage:>3.1f}%'), - ) - overall_progress = Progress( - TimeElapsedColumn(), BarColumn(bar_width=110), TextColumn('{task.description}') - ) - - workerjobs: list[TaskID] = [ - thread_progress.add_task(f'[bold blue][T {threadNum}]', total=100) - for threadNum in range(threadCount) - ] - overalljob = overall_progress.add_task('(0 of ?)', total=100) - - with Live( - Group( - Panel(thread_progress), - overall_progress, - ), - refresh_per_second=10, - ): - completed = 0 - while completed != threadCount: - i = 0 - completed = 0 - progressAvg = 0 - - for jobID in workerjobs: - jobProgress = newProcess._threads[i].progress - thread_progress.update(jobID, completed=round(jobProgress * 100, 2)) - if jobProgress == 1: - thread_progress.stop_task(jobID) - thread_progress.update(jobID, description='[bold green]Completed') - completed += 1 - - progressAvg += jobProgress - i += 1 - - # Update overall - overall_progress.update( - overalljob, - description=f'[bold {"green" if completed == threadCount else "#AAAAAA"}]({completed} of {threadCount})', - completed=round((progressAvg / threadCount) * 100, 2), - ) - time.sleep(0.1) - - result = newProcess.get_return_values() - - typer.echo(f'Completed in {(time.perf_counter() - start_t):.5f}s') - if fileout: - typer.echo(f'Writing to {output}') - try: - with open(output, 'w') as f: - json.dump(result, f, indent=2) - logger.info('Wrote to file') - except Exception as e: - logger.error('Failed to write to file') - logger.debug(str(e)) - - if stdout: - typer.echo(result) diff --git a/src/thread-cli/utils/__init__.py b/src/thread-cli/utils/__init__.py deleted file mode 100644 index 30edbd9..0000000 --- a/src/thread-cli/utils/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from . import logging -from .processors import ( - verbose_args_processor, - kwargs_processor, - DebugOption, - VerboseOption, - QuietOption, -) diff --git a/src/thread-cli/utils/logging.py b/src/thread-cli/utils/logging.py deleted file mode 100644 index 1d598a8..0000000 --- a/src/thread-cli/utils/logging.py +++ /dev/null @@ -1,39 +0,0 @@ -import logging -from colorama import init, Fore, Style - -init(autoreset=True) - - -# Stdout color configuration -class ColorFormatter(logging.Formatter): - COLORS = { - 'DEBUG': Fore.BLUE, - 'INFO': Fore.GREEN, - 'WARNING': Fore.YELLOW, - 'ERROR': Fore.RED, - 'CRITICAL': Fore.RED + Style.BRIGHT, - } - - def format(self, record): - color = self.COLORS.get(record.levelname, '') - record.levelname = ( - record.levelname - if not color - else color + Style.BRIGHT + f'{record.levelname:<9}|' - ) - record.msg = ( - record.msg if not color else color + Fore.WHITE + Style.NORMAL + record.msg - ) - - return logging.Formatter.format(self, record) - - -# Configure logger -class ColorLogger(logging.Logger): - def __init__(self, name): - logging.Logger.__init__(self, name, logging.DEBUG) - colorFormatter = ColorFormatter('%(levelname)s %(message)s' + Style.RESET_ALL) - - console = logging.StreamHandler() - console.setFormatter(colorFormatter) - self.addHandler(console) diff --git a/src/thread-cli/utils/processors.py b/src/thread-cli/utils/processors.py deleted file mode 100644 index 75c3ab3..0000000 --- a/src/thread-cli/utils/processors.py +++ /dev/null @@ -1,41 +0,0 @@ -# Verbose Command Processor # -import typer -import logging - - -# Verbose Options # -DebugOption = typer.Option( - False, '--debug', help='Set verbosity level to [blue]DEBUG[/blue]', is_flag=True -) -VerboseOption = typer.Option( - False, - '--verbose', - '-v', - help='Set verbosity level to [green]INFO[/green]', - is_flag=True, -) -QuietOption = typer.Option( - False, '--quiet', '-q', help='Set verbosity level to [red]ERROR[/red]', is_flag=True -) - - -# Helper functions # - - -# Processors # -def verbose_args_processor(debug: bool, verbose: bool, quiet: bool): - """Handles setting and raising exceptions for verbose""" - if verbose and quiet: - raise typer.BadParameter('--quiet cannot be used with --verbose') - - if verbose and debug: - raise typer.BadParameter('--debug cannot be used with --verbose') - - logging.getLogger('base').setLevel( - ((debug and logging.DEBUG) or (verbose and logging.INFO) or logging.ERROR) - ) - - -def kwargs_processor(arguments: list[str]) -> dict[str, str]: - """Processes arguments into kwargs""" - return {kwarg[0]: kwarg[1] for i in arguments if (kwarg := i.split('='))} diff --git a/src/thread-cli/__init__.py b/src/thread_cli/__init__.py similarity index 69% rename from src/thread-cli/__init__.py rename to src/thread_cli/__init__.py index 5085c14..caf8cbb 100644 --- a/src/thread-cli/__init__.py +++ b/src/thread_cli/__init__.py @@ -5,12 +5,14 @@ --- -Released under the GPG-3 License +Released under the BSD-3 License Copyright (c) 2020, thread.ngjx.org. All rights reserved. """ __version__ = '0.1.1' + +from . import utils from .utils.logging import ColorLogger, logging # Export Core @@ -18,9 +20,9 @@ from .process import process as process_cli app.command( - name='process', - no_args_is_help=True, - context_settings={'allow_extra_args': True}, + name='process', + no_args_is_help=True, + context_settings={'allow_extra_args': True}, )(process_cli) @@ -29,4 +31,4 @@ # Wildcard export -__all__ = ['app'] +__all__ = ['app', 'utils'] diff --git a/src/thread_cli/base.py b/src/thread_cli/base.py new file mode 100644 index 0000000..6846093 --- /dev/null +++ b/src/thread_cli/base.py @@ -0,0 +1,111 @@ +import typer +import logging + +from . import __version__ +from .utils import DebugOption, VerboseOption, QuietOption, verbose_args_processor + +logger = logging.getLogger('base') + + +cli_base = typer.Typer( + no_args_is_help=True, + rich_markup_mode='rich', + context_settings={'help_option_names': ['-h', '--help', 'help']}, +) + + +def version_callback(value: bool): + if value: + typer.echo(f'v{__version__}') + raise typer.Exit() + + +@cli_base.callback(invoke_without_command=True) +def callback( + version: bool = typer.Option( + None, + '--version', + callback=version_callback, + help='Get the current installed version', + is_eager=True, + ), + debug: bool = DebugOption, + verbose: bool = VerboseOption, + quiet: bool = QuietOption, +): + """ + [b]Thread CLI[/b]\b\n + [white]Use thread from the terminal![/white] + + [blue][u] [/u][/blue] + + Learn more from our [link=https://github.com/python-thread/thread/blob/main/docs/command-line.md]documentation![/link] + """ + verbose_args_processor(debug, verbose, quiet) + + +# Help and Others +@cli_base.command(rich_help_panel='Help and Others') +def help(): + """Get [yellow]help[/yellow] from the community. :question:""" + typer.echo('Feel free to search for or ask questions here!') + try: + logger.info('Attempting to open in web browser...') + + import webbrowser + + webbrowser.open('https://github.com/python-thread/thread/issues', new=2) + typer.echo('Opening in web browser!') + + except Exception as e: + logger.warn('Failed to open web browser') + logger.debug(f'{e}') + typer.echo('https://github.com/python-thread/thread/issues') + + +@cli_base.command(rich_help_panel='Help and Others') +def docs(): + """View our [yellow]documentation.[/yellow] :book:""" + typer.echo('Thanks for using Thread, here is our documentation!') + try: + logger.info('Attempting to open in web browser...') + import webbrowser + + webbrowser.open( + 'https://github.com/python-thread/thread/blob/main/docs/command-line.md', + new=2, + ) + typer.echo('Opening in web browser!') + + except Exception as e: + logger.warn('Failed to open web browser') + logger.debug(f'{e}') + typer.echo( + 'https://github.com/python-thread/thread/blob/main/docs/command-line.md' + ) + + +@cli_base.command(rich_help_panel='Help and Others') +def report(): + """[yellow]Report[/yellow] an issue. :bug:""" + typer.echo('Sorry you run into an issue, report it here!') + try: + logger.info('Attempting to open in web browser...') + import webbrowser + + webbrowser.open('https://github.com/python-thread/thread/issues', new=2) + typer.echo('Opening in web browser!') + + except Exception as e: + logger.warn('Failed to open web browser') + logger.debug(f'{e}') + typer.echo('https://github.com/python-thread/thread/issues') + + +# Utils and Configs +@cli_base.command(rich_help_panel='Utils and Configs') +def config(configuration: str): + """ + [blue]Configure[/blue] the system. :wrench: + """ + typer.echo('Coming soon!') diff --git a/src/thread_cli/process.py b/src/thread_cli/process.py new file mode 100644 index 0000000..1f70971 --- /dev/null +++ b/src/thread_cli/process.py @@ -0,0 +1,267 @@ +"""Parallel Processing command""" + +import os +import time +import json +import inspect +import importlib + +import typer +import logging +from typing import Union + +from rich.live import Live +from rich.panel import Panel +from rich.console import Group +from rich.progress import ( + Progress, + TaskID, + SpinnerColumn, + TextColumn, + BarColumn, + TimeRemainingColumn, + TimeElapsedColumn, +) +from .utils import ( + DebugOption, + VerboseOption, + QuietOption, + verbose_args_processor, + kwargs_processor, +) + +logger = logging.getLogger('base') + + +def process( + func: str = typer.Argument( + help='[blue].path.to.file[/blue]:[blue]function_name[/blue] OR [blue]lambda x: x[/blue]' + ), + dataset: str = typer.Argument( + help='[blue]./path/to/file.txt[/blue] OR [blue][ i for i in range(2) ][/blue]' + ), + args: list[str] = typer.Option( + [], '--arg', '-a', help='[blue]Arguments[/blue] passed to each thread' + ), + kargs: list[str] = typer.Option( + [], + '--kwarg', + '-kw', + help='[blue]Key-Value arguments[/blue] passed to each thread', + ), + threads: int = typer.Option( + 8, + '--threads', + '-t', + help='Maximum number of [blue]threads[/blue] (will scale down based on dataset size)', + ), + daemon: bool = typer.Option( + False, '--daemon', '-d', help='Threads to run in [blue]daemon[/blue] mode' + ), + graceful_exit: bool = typer.Option( + True, + '--graceful-exit', + '-ge', + is_flag=True, + help='Whether to [blue]gracefully exit[/blue] on abrupt exit (etc. CTRL+C)', + ), + output: str = typer.Option( + './output.json', '--output', '-o', help='[blue]Output[/blue] file location' + ), + fileout: bool = typer.Option( + True, + '--fileout', + is_flag=True, + help='Whether to [blue]write[/blue] output to a file', + ), + stdout: bool = typer.Option( + False, '--stdout', is_flag=True, help='Whether to [blue]print[/blue] the output' + ), + debug: bool = DebugOption, + verbose: bool = VerboseOption, + quiet: bool = QuietOption, +): + """ + [bold]Utilise parallel processing on a dataset[/bold] + + \b\n + [bold white]:glowing_star: Important[/bold white] + Args and Kwargs can be parsed by adding multiple -a or -kw + + [green]$ thread[/green] [blue]process[/blue] ... -a 'an arg' -kw myKey=myValue -arg testing --kwarg a1=a2 + [white]=> args = [ [green]'an arg'[/green], [green]'testing'[/green] ][/white] + [white] kwargs = { [green]'myKey'[/green]: [green]'myValue'[/green], [green]'a1'[/green]: [green]'a2'[/green] }[/white] + + [blue][u] [/u][/blue] + + Learn more from our [link=https://github.com/python-thread/thread/blob/main/docs/command-line.md#parallel-processing-thread-process]documentation![/link] + """ + verbose_args_processor(debug, verbose, quiet) + kwargs = kwargs_processor(kargs) + logger.debug('Processed kwargs: %s' % kwargs) + + # Verify output + if not fileout and not stdout: + raise typer.BadParameter('No output method specified') + + if fileout and not os.path.exists('/'.join(output.split('/')[:-1])): + raise typer.BadParameter('Output file directory does not exist') + + # Loading function + f = None + try: + logger.info('Attempted to interpret function') + f = eval( + func + ) # I know eval is bad practice, but I have yet to find a safer replacement + logger.debug('Evaluated function: %s' % f) + + if not inspect.isfunction(f): + logger.info('Invalid function') + except Exception: + logger.info('Failed to interpret function') + + if not f: + try: + logger.info('Attempting to fetch function file') + + fPath, fName = func.split(':') + f = importlib.import_module(fPath).__dict__[fName] + logger.debug('Evaluated function: %s' % f) + + if not inspect.isfunction(f): + logger.info('Not a function') + raise Exception('Not a function') + except Exception as e: + logger.warning('Failed to fetch function') + raise typer.BadParameter('Failed to fetch function') from e + + # Loading dataset + ds: Union[list, tuple, set, None] = None + try: + logger.info('Attempting to interpret dataset') + ds = eval(dataset) + logger.debug( + 'Evaluated dataset: %s' + % (str(ds)[:125] + '...' if len(str(ds)) > 125 else ds) + ) + + if not isinstance(ds, (list, tuple, set)): + logger.info('Invalid dataset literal') + ds = None + + except Exception: + logger.info('Failed to interpret dataset') + + if not ds: + try: + logger.info('Attempting to fetch data file') + if not os.path.isfile(dataset): + logger.info('Invalid file path') + raise Exception('Invalid file path') + + with open(dataset, 'r') as a: + ds = [i.endswith('\n') and i[:-2] for i in a.readlines()] + except Exception as e: + logger.warning('Failed to read dataset') + raise typer.BadParameter('Failed to read dataset') from e + + logger.info('Interpreted dataset') + + # Setup + logger.debug('Importing module') + from thread import Settings, ParallelProcessing + + logger.info( + 'Spawning threads... [Expected: {tcount} threads]'.format( + tcount=min(len(ds), threads) + ) + ) + + Settings.set_graceful_exit(graceful_exit) + newProcess = ParallelProcessing( + function=f, + dataset=list(ds), + args=args, + kwargs=kwargs, + daemon=daemon, + max_threads=threads, + ) + + logger.info('Created parallel process') + logger.info('Starting parallel process') + + start_t = time.perf_counter() + newProcess.start() + + logger.info('Started parallel processes') + typer.echo('Waiting for parallel processes to complete, this may take a while...') + + # Progress bar :D + threadCount = len(newProcess._threads) + + thread_progress = Progress( + SpinnerColumn(), + TextColumn('{task.description}'), + '•', + TimeRemainingColumn(), + BarColumn(bar_width=80), + TextColumn('{task.percentage:>3.1f}%'), + ) + overall_progress = Progress( + TimeElapsedColumn(), BarColumn(bar_width=110), TextColumn('{task.description}') + ) + + workerjobs: list[TaskID] = [ + thread_progress.add_task(f'[bold blue][T {threadNum}]', total=100) + for threadNum in range(threadCount) + ] + overalljob = overall_progress.add_task('(0 of ?)', total=100) + + with Live( + Group( + Panel(thread_progress), + overall_progress, + ), + refresh_per_second=10, + ): + completed = 0 + while completed != threadCount: + i = 0 + completed = 0 + progressAvg = 0 + + for jobID in workerjobs: + jobProgress = newProcess._threads[i].progress + thread_progress.update(jobID, completed=round(jobProgress * 100, 2)) + if jobProgress == 1: + thread_progress.stop_task(jobID) + thread_progress.update(jobID, description='[bold green]Completed') + completed += 1 + + progressAvg += jobProgress + i += 1 + + # Update overall + overall_progress.update( + overalljob, + description=f'[bold {"green" if completed == threadCount else "#AAAAAA"}]({completed} of {threadCount})', + completed=round((progressAvg / threadCount) * 100, 2), + ) + time.sleep(0.1) + + result = newProcess.get_return_values() + + typer.echo(f'Completed in {(time.perf_counter() - start_t):.5f}s') + if fileout: + typer.echo(f'Writing to {output}') + try: + with open(output, 'w') as f: + json.dump(result, f, indent=2) + logger.info('Wrote to file') + except Exception as e: + logger.error('Failed to write to file') + logger.debug(str(e)) + + if stdout: + typer.echo(result) diff --git a/src/thread-cli/py.typed b/src/thread_cli/py.typed similarity index 100% rename from src/thread-cli/py.typed rename to src/thread_cli/py.typed diff --git a/src/thread_cli/utils/__init__.py b/src/thread_cli/utils/__init__.py new file mode 100644 index 0000000..1ffcd5b --- /dev/null +++ b/src/thread_cli/utils/__init__.py @@ -0,0 +1,8 @@ +from . import logging +from .processors import ( + verbose_args_processor, + kwargs_processor, + DebugOption, + VerboseOption, + QuietOption, +) diff --git a/src/thread_cli/utils/logging.py b/src/thread_cli/utils/logging.py new file mode 100644 index 0000000..fe58999 --- /dev/null +++ b/src/thread_cli/utils/logging.py @@ -0,0 +1,39 @@ +import logging +from colorama import init, Fore, Style + +init(autoreset=True) + + +# Stdout color configuration +class ColorFormatter(logging.Formatter): + COLORS = { + 'DEBUG': Fore.BLUE, + 'INFO': Fore.GREEN, + 'WARNING': Fore.YELLOW, + 'ERROR': Fore.RED, + 'CRITICAL': Fore.RED + Style.BRIGHT, + } + + def format(self, record): + color = self.COLORS.get(record.levelname, '') + record.levelname = ( + record.levelname + if not color + else color + Style.BRIGHT + f'{record.levelname:<9}|' + ) + record.msg = ( + record.msg if not color else color + Fore.WHITE + Style.NORMAL + record.msg + ) + + return logging.Formatter.format(self, record) + + +# Configure logger +class ColorLogger(logging.Logger): + def __init__(self, name): + logging.Logger.__init__(self, name, logging.DEBUG) + colorFormatter = ColorFormatter('%(levelname)s %(message)s' + Style.RESET_ALL) + + console = logging.StreamHandler() + console.setFormatter(colorFormatter) + self.addHandler(console) diff --git a/src/thread_cli/utils/processors.py b/src/thread_cli/utils/processors.py new file mode 100644 index 0000000..0d9b0a9 --- /dev/null +++ b/src/thread_cli/utils/processors.py @@ -0,0 +1,81 @@ +# Verbose Command Processor # +import os +import typer +import logging + + +# Types # +SupportedFileExtensions = ['json', ''] + + +# Verbose Options # +DebugOption = typer.Option( + False, '--debug', help='Set verbosity level to [blue]DEBUG[/blue]', is_flag=True +) +VerboseOption = typer.Option( + False, + '--verbose', + '-v', + help='Set verbosity level to [green]INFO[/green]', + is_flag=True, +) +QuietOption = typer.Option( + False, '--quiet', '-q', help='Set verbosity level to [red]ERROR[/red]', is_flag=True +) + + +# Helper functions # + + +# Processors # +def verbose_args_processor(debug: bool, verbose: bool, quiet: bool): + """Handles setting and raising exceptions for verbose""" + if verbose and quiet: + raise typer.BadParameter('--quiet cannot be used with --verbose') + + if verbose and debug: + raise typer.BadParameter('--debug cannot be used with --verbose') + + logging.getLogger('base').setLevel( + ((debug and logging.DEBUG) or (verbose and logging.INFO) or logging.ERROR) + ) + + +def kwargs_processor(arguments: list[str]) -> dict[str, str]: + """Processes arguments into kwargs""" + return { + kwarg[0]: kwarg[1] + for i in arguments + if (kwarg := i.split('=')) and (len(kwarg) == 2) and kwarg[0] and kwarg[1] + } + + +def file_processor(file: str) -> list[str]: + """Processes file path or string representation of a python data structure""" + logger = logging.getLogger('base') + logger.debug('Attempting to process as file path') + + # Handle file path + try: + if not os.path.exists(file): + raise FileNotFoundError(f'File {file} does not exist') + + if not os.path.isfile(file): + raise FileNotFoundError(f'File {file} is not a file') + + with open(file, 'r') as f: + return f.readlines() + + except Exception as e: + logger.debug('Failed to process as file path: ' + str(e)) + + # Handle string + logger.debug('Attempting to read as python data structure') + try: + dataStruct = eval(file) + return dataStruct + except Exception as e: + logger.debug('Failed to process as python data structure: ' + str(e)) + + logger.error('Failed to process file') + exit(1) diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_kwargs_processing.py b/tests/test_kwargs_processing.py new file mode 100644 index 0000000..ab924fa --- /dev/null +++ b/tests/test_kwargs_processing.py @@ -0,0 +1,10 @@ +from thread_cli.utils import kwargs_processor + + +def test_normal(): + """Testing for normal arguments""" + assert kwargs_processor(['a=1', 'b=2', 'c=3', 'd=', 'e']) == { + 'a': '1', + 'b': '2', + 'c': '3', + } diff --git a/tests/test_placeholder.py b/tests/test_placeholder.py index d20e63f..aea6d5f 100644 --- a/tests/test_placeholder.py +++ b/tests/test_placeholder.py @@ -1,4 +1,4 @@ def test_packagesExist(): - import thread + import thread - assert thread + assert thread