diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 24f4aec..3c98a6f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -77,7 +77,7 @@ jobs: --cov-report xml - name: 'Upload coverage' - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: token: ${{ secrets.CODECOV_ORG_TOKEN }} @@ -140,6 +140,11 @@ jobs: env: PYMANAGER_DEBUG: true + - name: 'Install additional runtimes' + run: pymanager install 3.5 3.6 3.7 3.8 3.9 + env: + PYMANAGER_DEBUG: true + - name: 'List installed runtimes' run: pymanager list env: diff --git a/ci/release.yml b/ci/release.yml index 25b173b..943484b 100644 --- a/ci/release.yml +++ b/ci/release.yml @@ -251,6 +251,13 @@ stages: env: PYMANAGER_DEBUG: true + - powershell: | + pymanager install 3.5 3.6 3.7 3.8 3.9 + displayName: 'Install additional runtimes' + timeoutInMinutes: 10 + env: + PYMANAGER_DEBUG: true + - powershell: | pymanager list displayName: 'List installed runtimes' diff --git a/src/_native/winhttp.cpp b/src/_native/winhttp.cpp index 9ca3293..8d7d580 100644 --- a/src/_native/winhttp.cpp +++ b/src/_native/winhttp.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -108,7 +109,7 @@ static wchar_t **split_to_array(wchar_t *str, wchar_t sep) { } -static int crack_url(wchar_t *url, URL_COMPONENTS *parts, int add_nuls) { +static int crack_url(const wchar_t *url, URL_COMPONENTS *parts) { parts->lpszScheme = NULL; parts->lpszUserName = NULL; parts->lpszPassword = NULL; @@ -125,29 +126,66 @@ static int crack_url(wchar_t *url, URL_COMPONENTS *parts, int add_nuls) { winhttp_error(); return 0; } - if (add_nuls) { - if (parts->lpszScheme && parts->dwSchemeLength > 0) { - parts->lpszScheme[parts->dwSchemeLength] = L'\0'; - } - if (parts->lpszUserName && parts->dwUserNameLength > 0) { - parts->lpszUserName[parts->dwUserNameLength] = L'\0'; - } - if (parts->lpszPassword && parts->dwPasswordLength > 0) { - parts->lpszPassword[parts->dwPasswordLength] = L'\0'; - } - if (parts->lpszHostName && parts->dwHostNameLength > 0) { - parts->lpszHostName[parts->dwHostNameLength] = L'\0'; - } - if (parts->lpszUrlPath && parts->dwUrlPathLength > 0) { - parts->lpszUrlPath[parts->dwUrlPathLength] = L'\0'; + return 1; +} + +static wchar_t *_escape_url_part(bool encode, const wchar_t *url_part, DWORD cch, bool allow_env=false) +{ + if (!url_part) { + return NULL; + } + // Need to copy the incoming string to ensure it's null terminated + cch += 1; + wchar_t *url_string = (wchar_t *)PyMem_Malloc(sizeof(wchar_t) * cch); + if (!url_string) { + PyErr_NoMemory(); + return NULL; + } + wcsncpy_s(url_string, cch, url_part, cch - 1); + if (!url_string[0] || cch > 32767) { + // Too long/empty for the API, just bail out + return url_string; + } + if (allow_env && cch > 2 && url_string[0] == L'%' && url_string[cch - 2] == L'%') { + // Looks like an environment variable, so we won't change it. + return url_string; + } + + wchar_t *result = NULL; + HRESULT r = E_POINTER; + for (int retries = 3; retries > 0 && r == E_POINTER; --retries) { + result = (wchar_t *)PyMem_Realloc(result, sizeof(wchar_t) * cch); + if (!result) { + PyMem_Free(url_string); + PyErr_NoMemory(); + return NULL; } - if (parts->lpszExtraInfo && parts->dwExtraInfoLength > 0) { - parts->lpszExtraInfo[parts->dwExtraInfoLength] = L'\0'; + if (encode) { + // "SEGMENT_ONLY" means we want to escape the entire string + r = UrlEscapeW(url_string, result, &cch, URL_ESCAPE_SEGMENT_ONLY | URL_ESCAPE_ASCII_URI_COMPONENT); + } else { + r = UrlUnescapeW(url_string, result, &cch, 0); } } - return 1; + PyMem_Free(url_string); + if (r) { + err_SetFromWindowsErrWithMessage((DWORD)r); + return NULL; + } + return result; +} + +static wchar_t *escape_url_part(const wchar_t *url_part, DWORD cch, bool allow_env=false) +{ + return _escape_url_part(true, url_part, cch, allow_env); } +static wchar_t *unescape_url_part(const wchar_t *url_part, DWORD cch, bool allow_env=false) +{ + return _escape_url_part(false, url_part, cch, allow_env); +} + + extern "C" { #define CHECK_WINHTTP(x) if (!x) { winhttp_error(); goto exit; } @@ -226,7 +264,6 @@ static bool winhttp_apply_proxy(HINTERNET hSession, HINTERNET hRequest, const wc PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) { static const char * keywords[] = {"url", "method", "headers", "accepts", "chunksize", "on_progress", "on_cred_request", NULL}; wchar_t *url = NULL; - wchar_t *url2 = NULL; // a copy of url for splitting wchar_t *method = NULL; wchar_t *headers = NULL; wchar_t *accepts = NULL; @@ -246,7 +283,11 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) { uint64_t content_length; PyObject *chunks = NULL; uint64_t content_read = 0; - size_t n = 0; + + wchar_t *hostname = NULL; + wchar_t *urlpath = NULL; + wchar_t *user = NULL; + wchar_t *pass = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&O&O&O&|nOO:winhttp_urlopen", keywords, as_utf16, &url, as_utf16, &method, as_utf16, &headers, as_utf16, &accepts, &chunksize, &on_progress, &on_cred_request)) { @@ -264,17 +305,19 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) { if (!accepts_array) { goto exit; } - n = wcslen(url) + 1; - url2 = (wchar_t *)PyMem_Malloc(n * sizeof(wchar_t)); - if (!url2) { - PyErr_NoMemory(); + if (!crack_url(url, &url_parts)) { + goto exit; + } + hostname = unescape_url_part(url_parts.lpszHostName, url_parts.dwHostNameLength); + if (!hostname) { goto exit; } - wcscpy_s(url2, n, url); - if (!crack_url(url2, &url_parts, 1)) { + urlpath = unescape_url_part(url_parts.lpszUrlPath, url_parts.dwUrlPathLength); + if (!urlpath) { goto exit; } + hSession = WinHttpOpen( NULL, WINHTTP_ACCESS_TYPE_DEFAULT_PROXY, @@ -310,19 +353,16 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) { hConnection = WinHttpConnect( hSession, - url_parts.lpszHostName, + hostname, url_parts.nPort, 0 ); CHECK_WINHTTP(hConnection); - if (url_parts.dwUrlPathLength && !url_parts.lpszUrlPath[0]) { - url_parts.lpszUrlPath[0] = L'/'; - } hRequest = WinHttpOpenRequest( hConnection, method, - url_parts.lpszUrlPath, + urlpath && urlpath[0] ? urlpath : L"/", NULL, WINHTTP_NO_REFERER, accepts_array, @@ -341,12 +381,20 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) { )); if (url_parts.dwUserNameLength || url_parts.dwPasswordLength) { + user = unescape_url_part(url_parts.lpszUserName, url_parts.dwUserNameLength); + if (!user) { + goto exit; + } + pass = unescape_url_part(url_parts.lpszPassword, url_parts.dwPasswordLength); + if (!pass) { + goto exit; + } CHECK_WINHTTP(WinHttpSetCredentials( hRequest, WINHTTP_AUTH_TARGET_SERVER, WINHTTP_AUTH_SCHEME_BASIC, - url_parts.lpszUserName, - url_parts.lpszPassword, + user, + pass, NULL )); } @@ -464,11 +512,14 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) { if (hSession) { WinHttpCloseHandle(hSession); } + PyMem_Free(user); + PyMem_Free(pass); + PyMem_Free(hostname); + PyMem_Free(urlpath); PyMem_Free(accepts_array); PyMem_Free(accepts); PyMem_Free(headers); PyMem_Free(method); - PyMem_Free(url2); PyMem_Free(url); return result; } @@ -510,19 +561,26 @@ PyObject *winhttp_urlsplit(PyObject *, PyObject *args, PyObject *kwargs) { return NULL; } URL_COMPONENTS url_parts = { sizeof(URL_COMPONENTS) }; - if (!crack_url(url, &url_parts, 0)) { + if (!crack_url(url, &url_parts)) { PyMem_Free(url); return NULL; } + // Deliberately not decoding host or path. We never use a blacklist, we only + // match against values specified by the user, or pass it to. If they want + // to provide the same URL with different encoding, that's their fault. + wchar_t *user = unescape_url_part(url_parts.lpszUserName, url_parts.dwUserNameLength, true); + wchar_t *pass = unescape_url_part(url_parts.lpszPassword, url_parts.dwPasswordLength, true); PyObject *r = Py_BuildValue("(u#u#u#u#nu#u#)", url_parts.lpszScheme, (Py_ssize_t)url_parts.dwSchemeLength, - url_parts.lpszUserName, (Py_ssize_t)url_parts.dwUserNameLength, - url_parts.lpszPassword, (Py_ssize_t)url_parts.dwPasswordLength, + user, (Py_ssize_t)(user ? wcslen(user) : 0), + pass, (Py_ssize_t)(pass ? wcslen(pass) : 0), url_parts.lpszHostName, (Py_ssize_t)url_parts.dwHostNameLength, (Py_ssize_t)url_parts.nPort, url_parts.lpszUrlPath, (Py_ssize_t)url_parts.dwUrlPathLength, url_parts.lpszExtraInfo, (Py_ssize_t)url_parts.dwExtraInfoLength ); + PyMem_Free(user); + PyMem_Free(pass); PyMem_Free(url); return r; } @@ -532,10 +590,12 @@ PyObject *winhttp_urlunsplit(PyObject *, PyObject *args, PyObject *kwargs) { static const char * keywords[] = {"scheme", "user", "password", "netloc", "port", "path", "extra", NULL}; URL_COMPONENTS url = { sizeof(URL_COMPONENTS) }; Py_ssize_t port = 0; + wchar_t *user = NULL; + wchar_t *pass = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&O&O&O&nO&O&:winhttp_urlunsplit", keywords, as_utf16, &url.lpszScheme, - as_utf16, &url.lpszUserName, - as_utf16, &url.lpszPassword, + as_utf16, &user, + as_utf16, &pass, as_utf16, &url.lpszHostName, &port, as_utf16, &url.lpszUrlPath, @@ -545,8 +605,16 @@ PyObject *winhttp_urlunsplit(PyObject *, PyObject *args, PyObject *kwargs) { } DWORD cch = 0; PyObject *r = NULL; + url.lpszUserName = escape_url_part(user, user ? wcslen(user) : 0, true); + if (user && !url.lpszUserName) { + goto exit; + } + url.lpszPassword = escape_url_part(pass, pass ? wcslen(pass) : 0, true); + if (pass && !url.lpszPassword) { + goto exit; + } url.nPort = (INTERNET_PORT)port; - if (WinHttpCreateUrl(&url, ICU_ESCAPE, NULL, &cch)) { + if (WinHttpCreateUrl(&url, 0, NULL, &cch)) { // Success path, because it should've failed with ERROR_INSUFFICIENT_BUFFER PyErr_SetString(PyExc_ValueError, "unable to unsplit URL"); } else if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) { @@ -556,7 +624,7 @@ PyObject *winhttp_urlunsplit(PyObject *, PyObject *args, PyObject *kwargs) { wchar_t *buf = (wchar_t*)PyMem_Malloc(cch * sizeof(wchar_t)); if (!buf) { PyErr_NoMemory(); - } else if (!WinHttpCreateUrl(&url, ICU_ESCAPE, buf, &cch)) { + } else if (!WinHttpCreateUrl(&url, 0, buf, &cch)) { winhttp_error(); PyMem_Free(buf); } else { @@ -564,6 +632,9 @@ PyObject *winhttp_urlunsplit(PyObject *, PyObject *args, PyObject *kwargs) { PyMem_Free(buf); } } +exit: + PyMem_Free(user); + PyMem_Free(pass); PyMem_Free(url.lpszScheme); PyMem_Free(url.lpszUserName); PyMem_Free(url.lpszPassword); diff --git a/src/manage/aliasutils.py b/src/manage/aliasutils.py index 628d78e..ac2b7b4 100644 --- a/src/manage/aliasutils.py +++ b/src/manage/aliasutils.py @@ -3,7 +3,7 @@ from .exceptions import FilesInUseError, NoLauncherTemplateError from .fsutils import atomic_unlink, ensure_tree, unlink from .logging import LOGGER -from .pathutils import Path, relative_to +from .pathutils import Path, PurePath, relative_to from .tagutils import install_matches_any _EXE = ".exe".casefold() @@ -97,6 +97,10 @@ def _create_alias( allow_link=True, _link=os.link): p = cmd.global_dir / name + # Raise exception if someone has tried to get us to write outside of the + # intended directory + if str(p.relative_to(cmd.global_dir)) != PurePath(name).name: + raise ValueError(f"Invalid alias name: {name}") if not p.match("*.exe"): p = p.with_name(p.name + ".exe") if not isinstance(target, Path): @@ -235,6 +239,9 @@ def _parse_entrypoint_line(line): name, sep, rest = line.partition("=") name = name.strip() if name and name[0].isalnum() and sep and rest: + # "names" that have a parent directory/slash are invalid + if PurePath(name).parent: + return None, None, None mod, sep, rest = rest.partition(":") mod = mod.strip() if mod and sep and rest: @@ -382,6 +389,14 @@ def create_aliases(cmd, aliases, *, allow_link=True, _create_alias=_create_alias else: LOGGER.debug("Skipping %s alias because " "the launcher template was not found.", alias.name) + except Exception: + if install_matches_any(alias.install, getattr(cmd, "tags", None)): + LOGGER.warn("Skipping %s alias because an unexpected error " + "occurred.", alias.name) + LOGGER.debug("TRACEBACK", exc_info=True) + else: + LOGGER.debug("Skipping %s alias because an unexpected error " + "occurred.", alias.name, exc_info=True) diff --git a/src/manage/commands.py b/src/manage/commands.py index 4d5eafd..f437619 100644 --- a/src/manage/commands.py +++ b/src/manage/commands.py @@ -40,9 +40,8 @@ WELCOME = f"""!B!Python install manager was successfully updated to {__version__}.!W! -Indexes can now be signed to provide tamper detection. When an index signature -is found to be invalid, the operation will be aborted without modifying your system. -See !B!{HELP_URL}#index-signatures!W! for more information. +Additional shebang configuration is now available. Please see +!B!{HELP_URL}#shebang-lines!W! for more information. """ # The 'py help' or 'pymanager help' output is constructed by these default docs, @@ -242,6 +241,9 @@ def execute(self): "include_unmanaged": (config_bool, None, "env"), "shebang_can_run_anything": (config_bool, None, "env"), "shebang_can_run_anything_silently": (config_bool, None, "env"), + # Mapping from shebang template to '-V:Company/Tag' argument or an + # executable path. The latter requires 'shebang_can_run_anything'. + "shebang_templates": (dict, config_dict_merge), # Typically configured to '%VIRTUAL_ENV%' to pick up the active environment "virtual_env": (str, None, "env", "path"), @@ -347,6 +349,7 @@ class BaseCommand: virtual_env = None shebang_can_run_anything = True shebang_can_run_anything_silently = False + shebang_templates = {} welcome_on_update = False log_file = None @@ -366,7 +369,7 @@ class BaseCommand: launcher_exe = None launcherw_exe = None - source_settings = None + source_settings = {} show_help = False @@ -902,12 +905,25 @@ class UninstallCommand(BaseCommand): purge = False by_id = False - # Not settable, but are checked by update_all_shortcuts() so we need them. + # Not directly settable, but will be copied from install configuration enable_shortcut_kinds = None disable_shortcut_kinds = None + enable_entrypoints = True + hard_link_entrypoints = True def execute(self): from .uninstall_command import execute + # Copy settings from install section of config + for k in [ + "enable_shortcut_kinds", + "disable_shortcut_kinds", + "enable_entrypoints", + "hard_link_entrypoints", + ]: + v = self.config.get("install", {}).get(k, ...) + if v is not ...: + setattr(self, k, v) + LOGGER.debug("Setting config %s to %s from install.%s", k, v, k) self.show_welcome() execute(self) diff --git a/src/manage/install_command.py b/src/manage/install_command.py index 9fa1202..1795bea 100644 --- a/src/manage/install_command.py +++ b/src/manage/install_command.py @@ -48,7 +48,7 @@ def _expand_versions_by_tag(versions): yield {**v, "tag": t} -def select_package(index_downloader, tag, platform=None, *, urlopen=_urlopen, by_id=False): +def select_package(index_downloader, tag, platform=None, *, urlopen=_urlopen, by_id=False, allow_pre=True): """Finds suitable package from index.json that looks like: {"versions": [ {"id": ..., "company": ..., "tag": ..., "url": ..., "hash": {"sha256": hexdigest}}, @@ -63,6 +63,8 @@ def select_package(index_downloader, tag, platform=None, *, urlopen=_urlopen, by try: if by_id: for v in index.versions: + if not allow_pre and v["sort-version"].is_prerelease: + continue if v["id"].casefold() == tag.casefold(): return v raise LookupError("Could not find a runtime matching '{}' at '{}'".format( @@ -70,12 +72,22 @@ def select_package(index_downloader, tag, platform=None, *, urlopen=_urlopen, by )) if platform: try: - return index.find_to_install(tag + platform) + v = index.find_to_install(tag + platform) except LookupError: pass - return index.find_to_install(tag) + else: + if allow_pre or not v["sort-version"].is_prerelease: + return v + v = index.find_to_install(tag) + if v["sort-version"].is_prerelease and not allow_pre: + raise LookupError("Found '{}' matching '{}' but excluded " + "because we are now allowing prereleases.".format( + v["display-name"], tag)) + return v except LookupError as ex: - first_exc = ex + LOGGER.debug("Potential error: %s", ex) + if not first_exc: + first_exc = ex if first_exc: raise first_exc @@ -367,7 +379,7 @@ def _same_install(i, j): return i["id"] == j["id"] and i["sort-version"] == j["sort-version"] -def _find_one(cmd, source, tag, *, installed=None, by_id=False): +def _find_one(cmd, source, tag, *, installed=None, by_id=False, allow_pre=True): if by_id: LOGGER.debug("Searching for Python with ID %s", tag) elif tag: @@ -377,7 +389,13 @@ def _find_one(cmd, source, tag, *, installed=None, by_id=False): download_cache = cmd.scratch.setdefault("install_command.download_cache", {}) downloader = IndexDownloader(cmd, source, Index, {}, download_cache) - install = select_package(downloader, tag, cmd.default_platform, by_id=by_id) + install = select_package(downloader, tag, cmd.default_platform, by_id=by_id, allow_pre=allow_pre) + + # Ensure the requested source URL is in the install + if install and source: + if install.get("source") != source: + LOGGER.verbose("Storing %s as source of package", sanitise_url(source)) + install = {**install, "source": source} if by_id: return install @@ -823,14 +841,16 @@ def execute(cmd): # Fallthrough is safe - cmd.tags is empty elif cmd.update: LOGGER.verbose("No tags provided, updating all installs:") + from .verutils import Version for install in installed: first_exc = None update = None + allow_pre = Version(install['sort-version']).is_prerelease for source in [install.get('source'), cmd.source, cmd.fallback_source]: if not source: continue try: - update = _find_one(cmd, source, install['id'], by_id=True) + update = _find_one(cmd, source, install['id'], by_id=True, allow_pre=allow_pre) if update: break except LookupError: diff --git a/src/manage/pathutils.py b/src/manage/pathutils.py index f2798e8..fca08cc 100644 --- a/src/manage/pathutils.py +++ b/src/manage/pathutils.py @@ -83,7 +83,8 @@ def parts(self): while ".." in bits: i = bits.index("..") bits.pop(i) - bits.pop(i - 1) + if i >= 1: + bits.pop(i - 1) return bits def __truediv__(self, other): diff --git a/src/manage/scriptutils.py b/src/manage/scriptutils.py index d203d8c..9afaa63 100644 --- a/src/manage/scriptutils.py +++ b/src/manage/scriptutils.py @@ -121,7 +121,61 @@ def _find_on_path(cmd, full_cmd): } +def _replace_templates(cmd, line, windowed): + # Override can be the entire line or just the first argument + # Override can be the entire line (including args) or just the first argument + m = re.match(r"^#!\s*([^\s]+)(.*)$", line) + + if not m: + return None, None + + full_key = (m.group(1) + m.group(2)).strip() + if full_key in cmd.shebang_templates: + template_key = full_key + suffix = "" + elif m.group(1) in cmd.shebang_templates: + template_key = m.group(1) + suffix = m.group(2) + else: + return None, None + + new_cmd = cmd.shebang_templates[template_key] + LOGGER.verbose("Using '%s' from configuration file in place of shebang '%s'", + new_cmd, template_key) + install = None + if new_cmd.startswith("py -V:"): + install = cmd.get_install_to_run(new_cmd[6:], windowed=windowed) + elif new_cmd.startswith("pyw -V:"): + install = cmd.get_install_to_run(new_cmd[7:], windowed=True) + elif new_cmd.startswith("py -3"): + install = cmd.get_install_to_run(f"PythonCore/{new_cmd[4:]}", windowed=windowed) + elif new_cmd.startswith("pyw -3"): + install = cmd.get_install_to_run(f"PythonCore/{new_cmd[5:]}", windowed=True) + elif new_cmd == "py": + install = cmd.get_install_to_run(windowed=windowed) + elif new_cmd == "pyw": + install = cmd.get_install_to_run(windowed=True) + else: + # Recreate the shebang with the alternate command and continue. + line = f"#!{new_cmd}{suffix}" + return install, line + + def _parse_shebang(cmd, line, *, windowed=None): + # To silence our warning when we get the path from config file + run_anything_silently = False + + # First check the user-provided overrides + if cmd.shebang_templates: + install, new_line = _replace_templates(cmd, line, windowed) + if install: + return install + if new_line: + # We don't warn about custom executables if they've come from + # the config file, unless they don't exist or are disabled. + run_anything_silently = True + line = new_line + # For /usr[/local]/bin, we look for a matching alias name. shebang = re.match(r"#!\s*/usr/(?:local/)?bin/(?!env\b)([^\\/\s]+).*", line) if shebang: @@ -151,7 +205,7 @@ def _parse_shebang(cmd, line, *, windowed=None): # If not, warn and do regular PATH search if cmd.shebang_can_run_anything or cmd.shebang_can_run_anything_silently: i = _find_on_path(cmd, full_cmd) - if not cmd.shebang_can_run_anything_silently: + if not cmd.shebang_can_run_anything_silently and not run_anything_silently: LOGGER.warn("A shebang '%s' was found but could not be matched " "to an installed runtime, so it will be treated as " "an arbitrary command.", full_cmd) @@ -181,14 +235,20 @@ def _parse_shebang(cmd, line, *, windowed=None): except LookupError: pass if cmd.shebang_can_run_anything or cmd.shebang_can_run_anything_silently: - if not cmd.shebang_can_run_anything_silently: + if not cmd.shebang_can_run_anything_silently and not run_anything_silently: LOGGER.warn("A shebang '%s' was found but does not match any " - "supported template (e.g. '/usr/bin/python'), so it " - "will be treated as an arbitrary command.", full_cmd) + "supported or configured template (e.g. " + "'/usr/bin/python'), so it will be treated as an " + "arbitrary command.", full_cmd) LOGGER.warn("To prevent execution of programs that are not " "Python runtimes, set 'shebang_can_run_anything' to " "'false' in your configuration file.") - return _find_on_path(cmd, full_cmd) + try: + return _find_on_path(cmd, full_cmd) + except LookupError as ex: + LOGGER.error("Could not launch '%s'. Using default interpreter " + "instead.", full_cmd) + raise else: LOGGER.warn("A shebang '%s' was found, but could not be matched " "to an installed runtime.", full_cmd) diff --git a/src/manage/startutils.py b/src/manage/startutils.py index 20efcb5..e46a7bf 100644 --- a/src/manage/startutils.py +++ b/src/manage/startutils.py @@ -1,4 +1,5 @@ import _native +import time from .fsutils import rmtree, unlink from .logging import LOGGER @@ -47,15 +48,28 @@ def _make(root, prefix, item, allow_warn=True): LOGGER.debug("Path: %s", lnk) LOGGER.debug("Directory: %s", root) return None - _native.shortcut_create( - lnk, - target, - arguments=_unprefix(item.get("Arguments"), prefix), - working_directory=_unprefix(item.get("WorkingDirectory"), prefix) - or _native.shortcut_default_cwd(), - icon=_unprefix(item.get("Icon"), prefix), - icon_index=item.get("IconIndex", 0), - ) + first_exc = None + for retry in range(5): + try: + _native.shortcut_create( + lnk, + target, + arguments=_unprefix(item.get("Arguments"), prefix), + working_directory=_unprefix(item.get("WorkingDirectory"), prefix) + or _native.shortcut_default_cwd(), + icon=_unprefix(item.get("Icon"), prefix), + icon_index=item.get("IconIndex", 0), + ) + break + except OSError as ex: + # No errors are reasonably expected here, so we'll retry for any. + # So far, all we've observed have been race conditions. + if not first_exc: + first_exc = ex + time.sleep(0.1) + if first_exc: + LOGGER.debug("Failed to create shortcut") + raise first_exc return lnk diff --git a/src/manage/urlutils.py b/src/manage/urlutils.py index 540fd53..f88f0a1 100644 --- a/src/manage/urlutils.py +++ b/src/manage/urlutils.py @@ -552,9 +552,11 @@ def sanitise_url(url): if ex.winerror in (12005, 12006): return url raise - p[U_USERNAME] = None + u = p[U_USERNAME] + if u and not (u.startswith("%") and u.endswith("%")): + p[U_USERNAME] = None pw = p[U_PASSWORD] - if pw and not (pw.startswith("%") and pw.startswith("%")): + if pw and not (pw.startswith("%") and pw.endswith("%")): p[U_PASSWORD] = None return winhttp_urlunsplit(*p) diff --git a/src/pymanager.json b/src/pymanager.json index fc8c8a0..95ca599 100644 --- a/src/pymanager.json +++ b/src/pymanager.json @@ -34,6 +34,17 @@ "launcherw_exe": "./templates/launcherw.exe", "welcome_on_update": true, + "shebang_templates": { + "/usr/bin/python": "py", + "/usr/bin/pythonw": "pyw", + "/usr/bin/python3": "py", + "/usr/bin/pythonw3": "pyw", + "/usr/local/bin/python": "py", + "/usr/local/bin/pythonw": "pyw", + "/usr/local/bin/python3": "py", + "/usr/local/bin/pythonw3": "pyw" + }, + "source_settings": { "https://www.python.org/ftp/python/index-windows.json": { "requires_signature": true, diff --git a/tests/conftest.py b/tests/conftest.py index 32e5629..189129a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -162,6 +162,7 @@ def __init__(self, root, installs=[]): self.installs = list(installs) self.shebang_can_run_anything = True self.shebang_can_run_anything_silently = False + self.shebang_templates = {} self.scratch = {} def get_installs(self, *, include_unmanaged=False, set_default=True): @@ -169,7 +170,7 @@ def get_installs(self, *, include_unmanaged=False, set_default=True): return self.installs return [i for i in self.installs if not i.get("unmanaged", 0)] - def get_install_to_run(self, tag, *, windowed=False): + def get_install_to_run(self, tag=None, script=None, *, windowed=False): if windowed: i = self.get_install_to_run(tag) target = [t for t in i.get("run-for", []) if t.get("windowed")] @@ -177,13 +178,16 @@ def get_install_to_run(self, tag, *, windowed=False): return {**i, "executable": i["prefix"] / target[0]["target"]} return i - company, _, tag = tag.replace("/", "\\").rpartition("\\") - try: - found = [i for i in self.installs - if (not tag or i["tag"] == tag) and (not company or i["company"] == company)] - except LookupError as ex: - # LookupError is expected from this function, so make sure we don't raise it here - raise RuntimeError from ex + if not tag: + found = [i for i in self.installs if i.get("default")] + else: + company, _, tag = tag.replace("/", "\\").rpartition("\\") + try: + found = [i for i in self.installs + if (not tag or i["tag"] == tag) and (not company or i["company"] == company)] + except LookupError as ex: + # LookupError is expected from this function, so make sure we don't raise it here + raise RuntimeError from ex if found: return found[0] raise LookupError(tag) diff --git a/tests/test_alias.py b/tests/test_alias.py index b1da40d..d80c825 100644 --- a/tests/test_alias.py +++ b/tests/test_alias.py @@ -117,6 +117,17 @@ def test_write_script_alias(alias_checker): alias_checker.check_script(alias_checker.Cmd(), "1.0-32", "testA", windowed=0) +@pytest.mark.parametrize("name", [ + "..\\evil_path", + "dir\\..\\evil_path", + "normal\\subdir", + "C:\\absolute\\evil_path", +]) +def test_write_invalid_alias_name(alias_checker, name): + with pytest.raises(ValueError): + alias_checker.check(alias_checker.Cmd(), "1.0-32", name, None) + + def test_write_alias_launcher_missing(fake_config, assert_log, tmp_path): fake_config.launcher_exe = tmp_path / "non-existent.exe" fake_config.default_platform = '-32' @@ -255,6 +266,8 @@ def test_parse_entrypoint_line(): (" name = mod : func ", ("name", "mod", "func")), ("name=mod:func[extra]", ("name", "mod", "func")), ("name=mod:func [extra]", ("name", "mod", "func")), + ("../name=mod:func", (None, None, None)), + ("name/../../../name=mod:func", (None, None, None)), ]: assert expect == AU._parse_entrypoint_line(line) diff --git a/tests/test_install_command.py b/tests/test_install_command.py index a6940e2..bbb07b1 100644 --- a/tests/test_install_command.py +++ b/tests/test_install_command.py @@ -6,6 +6,7 @@ from manage import install_command as IC from manage import installs from manage.exceptions import NoInstallFoundError +from manage.logging import LOGGER def test_print_cli_shortcuts(patched_installs, assert_log, monkeypatch, tmp_path): @@ -225,50 +226,52 @@ def __init__(self, tmp_path, *args, **kwargs): self.scratch = { "install_command.download_cache": self.download_cache, } - self.automatic = kwargs.get("automatic", False) - self.by_id = kwargs.get("by_id", False) - self.default_install_tag = kwargs.get("default_install_tag", "1") - self.default_platform = kwargs.get("default_platform", "-32") - self.default_tag = kwargs.get("default_tag", "1") - self.download = kwargs.get("download") + self.automatic = kwargs.pop("automatic", False) + self.by_id = kwargs.pop("by_id", False) + self.default_install_tag = kwargs.pop("default_install_tag", "1") + self.default_platform = kwargs.pop("default_platform", "-32") + self.default_tag = kwargs.pop("default_tag", "1") + self.download = kwargs.pop("download", None) if self.download: self.download = tmp_path / self.download - self.download_dir = tmp_path / kwargs.get("download_dir", "_cache") - self.dry_run = kwargs.get("dry_run", True) - self.fallback_source = kwargs.get("fallback_source") - self.force = kwargs.get("force", True) - self.from_script = kwargs.get("from_script") - self.log_file = kwargs.get("log_file") - self.refresh = kwargs.get("refresh", False) - self.repair = kwargs.get("repair", False) - self.shebang_can_run_anything = kwargs.get("shebang_can_run_anything", False) - self.shebang_can_run_anything_silently = kwargs.get("shebang_can_run_anything_silently", False) - self.source = kwargs.get("source", "http://example.com/index.json") - self.target = kwargs.get("target") + self.download_dir = tmp_path / kwargs.pop("download_dir", "_cache") + self.dry_run = kwargs.pop("dry_run", True) + self.fallback_source = kwargs.pop("fallback_source", None) + self.force = kwargs.pop("force", True) + self.from_script = kwargs.pop("from_script", None) + self.log_file = kwargs.pop("log_file", None) + self.refresh = kwargs.pop("refresh", False) + self.repair = kwargs.pop("repair", False) + self.shebang_can_run_anything = kwargs.pop("shebang_can_run_anything", False) + self.shebang_can_run_anything_silently = kwargs.pop("shebang_can_run_anything_silently", False) + self.shebang_templates = {} + self.source = kwargs.pop("source", "http://example.com/index.json") + self.target = kwargs.pop("target", None) if self.target: self.target = tmp_path / self.target - self.update = kwargs.get("update", False) - self.virtual_env = kwargs.get("virtual_env") + self.update = kwargs.pop("update", False) + self.virtual_env = kwargs.pop("virtual_env", None) self.index_installs = [ + *kwargs.pop("index_installs", ()), { "schema": 1, - "id": "test-1.1-32", + "id": "test-32", "sort-version": "1.1", "company": "Test", "tag": "1.1-32", - "install-for": ["1", "1.1", "1.1-32"], + "install-for": ["1-32", "1.1-32", "1.1-32"], "display-name": "Test 1.1 (32)", "executable": "test.exe", "url": "about:blank", }, { "schema": 1, - "id": "test-1.0-32", + "id": "test-32", "sort-version": "1.0", "company": "Test", "tag": "1.0-32", - "install-for": ["1", "1.0", "1.0-32"], + "install-for": ["1-32", "1.0-32", "1.0-32"], "display-name": "Test 1.0 (32)", "executable": "test.exe", "url": "about:blank", @@ -280,8 +283,13 @@ def __init__(self, tmp_path, *args, **kwargs): self.installs = [{ **self.index_installs[-1], "source": self.source, - "prefix": tmp_path / "test-1.0-32", + "prefix": tmp_path / "test-32", }] + assert not kwargs + + def ask_yn(self, prompt, *args): + LOGGER.info(prompt, *args) + return True def get_log_file(self): return self.log_file @@ -302,6 +310,7 @@ def test_install_simple(tmp_path, assert_log): IC.execute(cmd) assert_log( assert_log.skip_until("Searching for Python matching %s", ["1.1"]), + assert_log.skip_until(".*Your existing %s install.*", ["Test 1.0 (32)", "Test 1.1 (32)"]), assert_log.skip_until("Installing %s", ["Test 1.1 (32)"]), ("Tag: %s\\\\%s", ["Test", "1.1-32"]), ) @@ -317,6 +326,58 @@ def test_install_already_installed(tmp_path, assert_log): ) + +def test_install_update(tmp_path, assert_log): + cmd = InstallCommandTestCmd(tmp_path, update=True, force=False, + index_installs=[ + {"schema": 1, "id": "test-32", "sort-version": "1.2rc1", + "display-name": "Test 1.2rc1 (32)", + "company": "Test", "tag": "1.2rc-32", + "install-for": ["1-32", "1.2-32", "1.2rc1-32"]} + ], + ) + + IC.execute(cmd) + assert_log( + assert_log.skip_until("Searching for Python with ID %s", ["test-32"]), + assert_log.skip_until("Updating to %s", ["Test 1.1 (32)"]), + ) + + +def test_install_update_explicit(tmp_path, assert_log): + cmd = InstallCommandTestCmd(tmp_path, "1", update=True, force=False, + index_installs=[ + {"schema": 1, "id": "test-32", "sort-version": "1.2rc1", + "display-name": "Test 1.2rc1 (32)", + "company": "Test", "tag": "1.2rc-32", + "install-for": ["1-32", "1.2-32", "1.2rc1-32"]} + ], + ) + + IC.execute(cmd) + assert_log( + assert_log.skip_until("Searching for Python matching %s", ["1"]), + assert_log.skip_until("Updating to %s", ["Test 1.1 (32)"]), + ) + + +def test_install_update_explicit_pre(tmp_path, assert_log): + cmd = InstallCommandTestCmd(tmp_path, "1rc", update=True, force=False, + index_installs=[ + {"schema": 1, "id": "test-32", "sort-version": "1.2rc1", + "display-name": "Test 1.2rc1 (32)", + "company": "Test", "tag": "1.2rc-32", + "install-for": ["1-32", "1.2-32", "1.2rc1-32"]} + ], + ) + + IC.execute(cmd) + assert_log( + assert_log.skip_until("Searching for Python matching %s", ["1rc"]), + assert_log.skip_until("Updating to %s", ["Test 1.2rc1 (32)"]), + ) + + def test_install_from_script(tmp_path, assert_log): cmd = InstallCommandTestCmd(tmp_path, from_script=tmp_path / "t.py") @@ -517,3 +578,33 @@ class Cmd: assert i["url"] == test_url_2 assert i["data1"] == "a" assert i["data2"] == "c" + + +def test_user_provided_source_retained(tmp_path, assert_log, monkeypatch): + cmd = InstallCommandTestCmd(tmp_path, "1.1", force=False) + + source1 = cmd.source + source2 = "http://example.com/index-2.json" + cmd.source = source2 + cmd.download_cache[source2] = json.dumps({ + "versions": [], + "next": source1, + }) + + def find_one(*args, _orig=IC._find_one, **kwargs): + i = _orig(*args, **kwargs) + assert i["source"] == cmd.source + return i + + monkeypatch.setattr(IC, "_find_one", find_one) + + IC.execute(cmd) + assert_log( + assert_log.skip_until("Searching for Python matching %s", ["1.1"]), + assert_log.skip_until("Fetching: %s", [source2]), + assert_log.skip_until("No install found.+"), + assert_log.skip_until("Fetching: %s", [source1]), + assert_log.skip_until("Storing %s as source of package", [source2]), + assert_log.skip_until("Installing %s", ["Test 1.1 (32)"]), + ("Tag: %s\\\\%s", ["Test", "1.1-32"]), + ) diff --git a/tests/test_scriptutils.py b/tests/test_scriptutils.py index 9ccf506..f281486 100644 --- a/tests/test_scriptutils.py +++ b/tests/test_scriptutils.py @@ -6,10 +6,13 @@ from pathlib import PurePath +import manage.scriptutils as SU from manage.scriptutils import ( find_install_from_script, _find_shebang_command, + _parse_shebang, _read_script, + _replace_templates, NewEncoding, _maybe_quote, quote_args, @@ -274,3 +277,61 @@ def test_quote_args(args, expect): assert expect == quote_args(args) # Test that our split function produces the same result assert args == split_args(expect), expect + + +@pytest.mark.parametrize("line, expect_id, expect_line", [pytest.param(*a, id=a[0]) for a in [ + ("#!/usr/bin/python", "Test1", None), + ("#!/usr/bin/python -Es", "Test1", None), + ("#! /usr/bin/pythonw", "Test1", None), + ("#! /usr/bin/python2", "Test2", None), + ("#! /usr/bin/pythonw2", "Test2", None), + ("#! /usr/bin/python3", "PythonCore3", None), + ("#! /usr/bin/pythonw3", "PythonCore3", None), + ("#! custom", None, "#!CUSTOM"), + ("#! full line custom", None, "#!CUSTOM2"), + ("#!full line custom with extra", None, None), + ("custom", None, None), + ("full line custom", None, None), +]]) +def test_shebang_templates(fake_config, line, expect_id, expect_line): + fake_config.installs = [ + dict(id="Test1", company="Test", tag="1", default=True), + dict(id="Test2", company="Test", tag="2"), + dict(id="Test3", company="Test", tag="3.2"), + dict(id="PythonCore3", company="PythonCore", tag="3.2"), + ] + fake_config.shebang_templates = { + "/usr/bin/python": "py", + "/usr/bin/pythonw": "pyw", + "/usr/bin/python2": "py -V:Test/2", + "/usr/bin/pythonw2": "pyw -V:Test/2", + "/usr/bin/python3": "py -3.2", + "/usr/bin/pythonw3": "pyw -3.2", + "custom": "CUSTOM", + "full line custom": "CUSTOM2", + } + actual, actual_line = _replace_templates(fake_config, line, False) + if expect_id: + assert actual + assert expect_id == actual["id"] + elif expect_line: + assert expect_line == actual_line + else: + assert not actual + assert not actual_line + + +def test_parse_shebang_templates(monkeypatch): + class Cmd: + shebang_templates = True + + expect = {"an": "install"} + monkeypatch.setattr(SU, "_replace_templates", lambda *a: (expect, None)) + actual = _parse_shebang(Cmd, "Anything at all") + assert expect == actual + + expect = {"id": "COMMAND"} + monkeypatch.setattr(SU, "_replace_templates", lambda *a: (None, "#!COMMAND")) + monkeypatch.setattr(SU, "_find_shebang_command", lambda cmd, full_cmd, **kw: {"id": full_cmd}) + actual = _parse_shebang(Cmd, "Anything at all", windowed=False) + assert expect == actual diff --git a/tests/test_urlutils.py b/tests/test_urlutils.py index 926bfe1..289856a 100644 --- a/tests/test_urlutils.py +++ b/tests/test_urlutils.py @@ -11,6 +11,8 @@ ("https://example.com/", "https://example.com/"), ("https://user@example.com/", "https://example.com/"), ("https://user:placeholder@example.com/", "https://example.com/"), + ("https://%placeholder%@example.com/", "https://%placeholder%@example.com/"), + ("https://%user%:%placeholder%@example.com/", "https://%user%:%placeholder%@example.com/"), ]]) def test_urlsanitise(url, expect): assert expect == UU.sanitise_url(url) @@ -30,10 +32,20 @@ def test_urlunsanitise(): assert None == UU.unsanitise_url(url, candidates) +def test_urlunsanitise_encoded(): + candidates = ["https://user%40example.com:place%40holder@example.com/"] + url = "https://example.com/my_path" + expect = "https://user%40example.com:place%40holder@example.com/my_path" + assert expect == UU.unsanitise_url(url, candidates) + + def test_extract_url_auth(): assert "1", "2" == UU.extract_url_auth("https://1:2@example.com") assert "1", "" == UU.extract_url_auth("https://1@example.com") + assert ("1", "2") == UU.extract_url_auth("https://%31:%32@example.com") + assert ("1", "") == UU.extract_url_auth("https://%31@example.com") + os.environ["PYMANAGER_TEST_VALUE"] = v = str(time.time()) assert "1", v == UU.extract_url_auth("https://1:%PYMANAGER_TEST_VALUE%@example.com") @@ -48,6 +60,11 @@ def test_extract_url_auth(): ("https://example.com/A/B/C", "//EXAMPLE.COM/A", True, "https://EXAMPLE.COM/A"), ("https://example.com/A/B/C", "//EXAMPLE.COM/", None, "https://EXAMPLE.COM/"), + # We are intentionally blind to encoded chars. + ("https://example.com/A/B/C", "%2fD", False, "https://example.com/A/B/C/%2fD"), + ("https://example.com/A/B/C", "%2f%2fD", False, "https://example.com/A/B/C/%2f%2fD"), + ("https://example.com/A/B%2fC", "D", True, "https://example.com/A/D"), + ("file:///C:/local/path", "file.json", False, "file:///C:/local/path/file.json"), ("file:///C:/local/path", "file.json", True, "file:///C:/local/file.json"), ("file:///C:/local/path", ".\\dir\\file.json", False, "file:///C:/local/path/dir/file.json"),