#!/usr/bin/env python3 import argparse import os import re import subprocess import sys import urllib.request import urllib.error ############################ # Built-in, Known Corrections, Exclusions ############################ BUILTIN_MODULES = { 'abc', 'aifc', 'argparse', 'array', 'ast', 'asynchat', 'asyncio', 'asyncore', 'atexit', 'audioop', 'base64', 'bdb', 'binascii', 'binhex', 'bisect', 'builtins', 'bz2', 'calendar', 'cgi', 'cgitb', 'chunk', 'cmath', 'cmd', 'code', 'codecs', 'codeop', 'collections', 'colorsys', 'compileall', 'concurrent', 'configparser', 'contextlib', 'copy', 'copyreg', 'crypt', 'csv', 'ctypes', 'curses', 'dataclasses', 'datetime', 'dbm', 'decimal', 'difflib', 'dis', 'distutils', 'doctest', 'dummy_threading', 'email', 'encodings', 'ensurepip', 'enum', 'errno', 'faulthandler', 'fcntl', 'filecmp', 'fileinput', 'fnmatch', 'formatter', 'fractions', 'ftplib', 'functools', 'gc', 'getopt', 'getpass', 'gettext', 'glob', 'gzip', 'hashlib', 'heapq', 'hmac', 'html', 'http', 'imaplib', 'imghdr', 'imp', 'importlib', 'inspect', 'io', 'ipaddress', 'itertools', 'json', 'keyword', 'lib2to3', 'linecache', 'locale', 'logging', 'lzma', 'mailbox', 'mailcap', 'marshal', 'math', 'mimetypes', 'modulefinder', 'multiprocessing', 'netrc', 'nntplib', 'numbers', 'operator', 'optparse', 'os', 'ossaudiodev', 'parser', 'pathlib', 'pdb', 'pickle', 'pickletools', 'pipes', 'pkgutil', 'platform', 'plistlib', 'poplib', 'posix', 'pprint', 'profile', 'pstats', 'pty', 'pwd', 'py_compile', 'pyclbr', 'pydoc', 'queue', 'quopri', 'random', 're', 'readline', 'reprlib', 'resource', 'rlcompleter', 'runpy', 'sched', 'secrets', 'select', 'selectors', 'shelve', 'shlex', 'shutil', 'signal', 'site', 'smtpd', 'smtplib', 'sndhdr', 'socket', 'socketserver', 'spwd', 'sqlite3', 'ssl', 'stat', 'statistics', 'string', 'stringprep', 'struct', 'subprocess', 'sunau', 'symtable', 'sys', 'sysconfig', 'syslog', 'tabnanny', 'tarfile', 'telnetlib', 'tempfile', 'termios', 'test', 'textwrap', 'threading', 'time', 'timeit', 'token', 'tokenize', 'trace', 'traceback', 'tracemalloc', 'tty', 'turtle', 'types', 'typing', 'unicodedata', 'unittest', 'urllib', 'uu', 'uuid', 'venv', 'warnings', 'wave', 'weakref', 'webbrowser', 'xdrlib', 'xml', 'xmlrpc', 'zipapp', 'zipfile', 'zipimport', 'zlib' } KNOWN_CORRECTIONS = { 'dateutil': 'python-dateutil', 'dotenv': 'python-dotenv', 'docx': 'python-docx', 'tesseract': 'pytesseract', 'magic': 'python-magic', 'multipart': 'python-multipart', 'newspaper': 'newspaper3k', 'srtm': 'elevation', 'yaml': 'pyyaml', 'zoneinfo': 'backports.zoneinfo' } EXCLUDED_NAMES = {'models', 'data', 'convert', 'example', 'tests'} ############################ # Environment & Installation ############################ def run_command(command): process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() return process.returncode, stdout.decode(), stderr.decode() def which(cmd): """ Check if `cmd` is on PATH. Returns True if found, else False. """ for pth in os.environ["PATH"].split(os.pathsep): cmd_path = os.path.join(pth, cmd) if os.path.isfile(cmd_path) and os.access(cmd_path, os.X_OK): return True return False def in_conda_env(): """ Returns True if we appear to be in a conda environment, typically indicated by CONDA_DEFAULT_ENV or other variables. """ return "CONDA_DEFAULT_ENV" in os.environ # We'll detect once at runtime (if in a conda env and skip_conda=False): # we either pick 'mamba' if available, else 'conda' if available, else None PREFERRED_CONDA_TOOL = None def detect_conda_tool(skip_conda=False): """ Decide which tool to use for conda-based installation: 1) If skip_conda is True or not in a conda env -> return None 2) If mamba is installed, return 'mamba' 3) Else if conda is installed, return 'conda' 4) Else return None """ if skip_conda or not in_conda_env(): return None if which("mamba"): return "mamba" elif which("conda"): return "conda" return None def is_package_installed(package, skip_conda=False): """ Checks if 'package' is installed with the chosen conda tool or pip. """ conda_tool = detect_conda_tool(skip_conda) if conda_tool == "mamba": returncode, stdout, _ = run_command(["mamba", "list"]) if returncode == 0: pattern = rf"^{re.escape(package)}\s" if re.search(pattern, stdout, re.MULTILINE): return True elif conda_tool == "conda": returncode, stdout, _ = run_command(["conda", "list"]) if returncode == 0: pattern = rf"^{re.escape(package)}\s" if re.search(pattern, stdout, re.MULTILINE): return True # Fall back to pip returncode, stdout, _ = run_command(["pip", "list"]) pattern = rf"^{re.escape(package)}\s" return re.search(pattern, stdout, re.MULTILINE) is not None def install_package(package, skip_conda=False): """ Installs 'package'. 1) Decide once if we can use 'mamba' or 'conda' (if skip_conda=False and in conda env). 2) Try that conda tool for installation 3) If that fails or not found, fallback to pip """ if is_package_installed(package, skip_conda=skip_conda): print(f"Package '{package}' is already installed.") return conda_tool = detect_conda_tool(skip_conda) if conda_tool == "mamba": print(f"Installing '{package}' with mamba...") returncode, _, _ = run_command(["mamba", "install", "-y", "-c", "conda-forge", package]) if returncode == 0: print(f"Successfully installed '{package}' via mamba.") return print(f"mamba failed for '{package}'. Falling back to pip...") elif conda_tool == "conda": print(f"Installing '{package}' with conda...") returncode, _, _ = run_command(["conda", "install", "-y", "-c", "conda-forge", package]) if returncode == 0: print(f"Successfully installed '{package}' via conda.") return print(f"conda failed for '{package}'. Falling back to pip...") # fallback: pip print(f"Installing '{package}' with pip...") returncode, _, _ = run_command(["pip", "install", package]) if returncode != 0: print(f"Failed to install package '{package}'.") else: print(f"Successfully installed '{package}' via pip.") ############################ # Parsing Python Imports ############################ def process_requirements_file(file_path): packages = set() with open(file_path, 'r') as file: for line in file: line = line.strip() if line and not line.startswith('#'): packages.add(line) return packages def process_python_file(file_path): """ Return a set of external imports (not built-in or excluded). Applies known corrections to recognized package names. """ imports = set() with open(file_path, 'r', encoding='utf-8', errors='ignore') as f: content = f.read() for line in content.split('\n'): line = line.strip() if line.startswith(('import ', 'from ')) and not line.startswith('#'): if line.startswith('import '): modules = line.replace('import ', '').split(',') for mod in modules: mod = re.sub(r'\s+as\s+\w+', '', mod).split('.')[0].strip() if mod and not mod.isupper() and mod not in EXCLUDED_NAMES and mod not in BUILTIN_MODULES: imports.add(KNOWN_CORRECTIONS.get(mod, mod)) elif line.startswith('from '): mod = line.split(' ')[1].split('.')[0].strip() if mod and not mod.isupper() and mod not in EXCLUDED_NAMES and mod not in BUILTIN_MODULES: imports.add(KNOWN_CORRECTIONS.get(mod, mod)) return imports def find_imports_in_path(path, recurse=False): """ Finds Python imports in the specified path. If path is a file, parse that file; if path is a dir, parse .py files in that dir. Recurse subdirs if 'recurse=True'. """ imports = set() if not os.path.exists(path): print(f"Warning: Path does not exist: {path}") return imports if os.path.isfile(path): if path.endswith('.py'): imports.update(process_python_file(path)) else: print(f"Skipping non-Python file: {path}") return imports # Directory: if recurse: for root, _, filenames in os.walk(path): for fn in filenames: if fn.endswith('.py'): imports.update(process_python_file(os.path.join(root, fn))) else: for fn in os.listdir(path): fullpath = os.path.join(path, fn) if os.path.isfile(fullpath) and fn.endswith('.py'): imports.update(process_python_file(fullpath)) return imports ############################ # PyPI Availability Check ############################ def check_library_on_pypi(library): """ Returns True if 'library' is on PyPI, else False. Using urllib to avoid external dependencies. """ url = f"https://pypi.org/pypi/{library}/json" try: with urllib.request.urlopen(url, timeout=5) as resp: return (resp.status == 200) # 200 => available except (urllib.error.URLError, urllib.error.HTTPError, ValueError): return False ############################ # Writing to requirements/missing ############################ def append_to_file(line, filename): """ Append 'line' to 'filename' only if it's not already in there. """ if not os.path.isfile(filename): with open(filename, 'w') as f: f.write(line + '\n') return with open(filename, 'r') as f: lines = {l.strip() for l in f.readlines() if l.strip()} if line not in lines: with open(filename, 'a') as f: f.write(line + '\n') ############################ # Subcommand: ls ############################ def subcmd_ls(parsed_args): """ Gathers imports, displays them, then writes them to requirements.txt or missing-packages.txt just like the original import_finder script did. """ path = parsed_args.path recurse = parsed_args.recurse imports = find_imports_in_path(path, recurse=recurse) if not imports: print("No Python imports found (or none that require external packages).") return print("Imports found:") for imp in sorted(imports): print(f" - {imp}") # Now we replicate the logic of import_finder: # If on PyPI => requirements.txt; else => missing-packages.txt for lib in sorted(imports): if check_library_on_pypi(lib): append_to_file(lib, 'requirements.txt') else: append_to_file(lib, 'missing-packages.txt') print("\nWrote results to requirements.txt (PyPI-available) and missing-packages.txt (unavailable).") ############################ # Subcommand: install ############################ def subcmd_install(parsed_args): """ If the user typed no direct packages/scripts or only used '-r' for recursion with no other args, we gather imports from the current dir, check PyPI availability, and install them with conda/mamba/pip (unless --no-conda is given). Otherwise, if the user typed e.g. '-R <reqfile>', or a .py file, or direct package names, we handle them. """ skip_conda = parsed_args.no_conda is_recursive = parsed_args.recurse # If user typed no leftover arguments or only the recursion flag, we do the "auto-scan & install" mode if not parsed_args.packages: # Means: "deps install" or "deps install -r" imports_found = find_imports_in_path('.', recurse=is_recursive) if not imports_found: print("No imports found in current directory.") return # Filter out those that are on PyPI to_install = [] for lib in sorted(imports_found): if check_library_on_pypi(lib): to_install.append(lib) else: print(f"Skipping '{lib}' (not found on PyPI).") if not to_install: print("No PyPI-available packages found to install.") return print("Installing packages:", ', '.join(to_install)) for pkg in to_install: install_package(pkg, skip_conda=skip_conda) return # Otherwise, we have leftover items: direct packages, .py files, or "-R" requirements. leftover_args = parsed_args.packages packages_to_install = set() i = 0 while i < len(leftover_args): arg = leftover_args[i] if arg == '-R': # next arg is a requirements file if i + 1 < len(leftover_args): req_file = leftover_args[i + 1] if os.path.isfile(req_file): pkgs = process_requirements_file(req_file) packages_to_install.update(pkgs) else: print(f"Requirements file not found: {req_file}") i += 2 else: print("Error: -R requires a file path.") return elif arg.endswith('.py'): # parse imports from that script if os.path.isfile(arg): pkgs = process_python_file(arg) packages_to_install.update(pkgs) else: print(f"File not found: {arg}") i += 1 else: # treat as a direct package name packages_to_install.add(arg) i += 1 # Now install them for pkg in sorted(packages_to_install): install_package(pkg, skip_conda=skip_conda) ############################ # Main ############################ def main(): parser = argparse.ArgumentParser(description='deps - Manage and inspect Python dependencies.') subparsers = parser.add_subparsers(dest='subcommand', required=True) # Subcommand: ls ls_parser = subparsers.add_parser( 'ls', help="List imports in a file/folder (and write them to requirements.txt/missing-packages.txt)." ) ls_parser.add_argument( '-r', '--recurse', action='store_true', help='Recurse into subfolders.' ) ls_parser.add_argument( 'path', nargs='?', default='.', help='File or directory to scan (default is current directory).' ) ls_parser.set_defaults(func=subcmd_ls) # Subcommand: install install_parser = subparsers.add_parser( 'install', help="Install packages or dependencies from .py files / current folder / subfolders." ) install_parser.add_argument( '-r', '--recurse', action='store_true', help="If no packages are specified, scanning current dir for imports will be recursive." ) install_parser.add_argument( '--no-conda', action='store_true', help="Skip using mamba/conda entirely and install only with pip." ) install_parser.add_argument( 'packages', nargs='*', help=( "Direct package names, .py files, or '-R <reqfile>'. If empty, scans current dir; " "if combined with -r, scans recursively. Example usage:\n" " deps install requests flask\n" " deps install script.py\n" " deps install -R requirements.txt\n" " deps install -r (recursively scan current dir)\n" ) ) install_parser.set_defaults(func=subcmd_install) parsed_args = parser.parse_args() parsed_args.func(parsed_args) if __name__ == "__main__": main()