From 35e972e3a41a5fa6d3f58259aa9c66ac8e2bed9e Mon Sep 17 00:00:00 2001 From: "Matthieu C." <886074+teto@users.noreply.github.com> Date: Mon, 17 Jun 2024 22:31:12 +0200 Subject: [PATCH] vimPluginsUpdater: make development easier `nix develop .#vimPluginsUpdater` now lets you enter a shell where you can run `pkgs/applications/editors/vim/plugins/update.py` and iteratively develop ! - removed `warn` warning from python by using `warning` instead - `plugin2nix` was calling the same bit of code over and over thus slowing down the generator by a lot --- maintainers/scripts/__init__.py | 0 maintainers/scripts/pluginupdate.py | 14 +++-- .../editors/vim/plugins/update.py | 56 +++++++++++-------- .../editors/vim/plugins/updater.nix | 16 +++--- .../lua-modules/updater/updater.py | 2 +- 5 files changed, 52 insertions(+), 36 deletions(-) create mode 100644 maintainers/scripts/__init__.py diff --git a/maintainers/scripts/__init__.py b/maintainers/scripts/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/maintainers/scripts/pluginupdate.py b/maintainers/scripts/pluginupdate.py index ce4f55ae1e3c..2e38023301a0 100644 --- a/maintainers/scripts/pluginupdate.py +++ b/maintainers/scripts/pluginupdate.py @@ -142,7 +142,7 @@ class Repo: return loaded def prefetch(self, ref: Optional[str]) -> str: - print("Prefetching") + print("Prefetching %s", self.uri) loaded = self._prefetch(ref) return loaded["sha256"] @@ -266,6 +266,7 @@ class PluginDesc: @staticmethod def load_from_csv(config: FetchConfig, row: Dict[str, str]) -> "PluginDesc": + log.debug("Loading row %s", row) branch = row["branch"] repo = make_repo(row["repo"], branch.strip()) repo.token = config.github_token @@ -328,7 +329,7 @@ def load_plugins_from_csv( -def run_nix_expr(expr, nixpkgs: str): +def run_nix_expr(expr, nixpkgs: str, **args): ''' :param expr nix expression to fetch current plugins :param nixpkgs Path towards a nixpkgs checkout @@ -347,7 +348,7 @@ def run_nix_expr(expr, nixpkgs: str): nix_path, ] log.debug("Running command: %s", " ".join(cmd)) - out = subprocess.check_output(cmd, timeout=90) + out = subprocess.check_output(cmd, **args) data = json.loads(out) return data @@ -736,6 +737,7 @@ def rewrite_input( redirects: Redirects = {}, append: List[PluginDesc] = [], ): + log.info("Rewriting input file %s", input_file) plugins = load_plugins_from_csv( config, input_file, @@ -744,10 +746,14 @@ def rewrite_input( plugins.extend(append) if redirects: + log.debug("Dealing with deprecated plugins listed in %s", deprecated) + cur_date_iso = datetime.now().strftime("%Y-%m-%d") with open(deprecated, "r") as f: deprecations = json.load(f) + # TODO parallelize this step for pdesc, new_repo in redirects.items(): + log.info("Rewriting input file %s", input_file) new_pdesc = PluginDesc(new_repo, pdesc.branch, pdesc.alias) old_plugin, _ = prefetch_plugin(pdesc) new_plugin, _ = prefetch_plugin(new_pdesc) @@ -791,7 +797,7 @@ def update_plugins(editor: Editor, args): start_time = time.time() redirects = update() duration = time.time() - start_time - print(f"The plugin update took {duration}s.") + print(f"The plugin update took {duration:.2f}s.") editor.rewrite_input(fetch_config, args.input_file, editor.deprecated, redirects) autocommit = not args.no_commit diff --git a/pkgs/applications/editors/vim/plugins/update.py b/pkgs/applications/editors/vim/plugins/update.py index e7e50c21d475..a8525b9018b5 100755 --- a/pkgs/applications/editors/vim/plugins/update.py +++ b/pkgs/applications/editors/vim/plugins/update.py @@ -40,7 +40,9 @@ ROOT = Path(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe import pluginupdate import importlib from pluginupdate import run_nix_expr, PluginDesc -import treesitter + +treesitter = importlib.import_module('nvim-treesitter.update') + HEADER = ( @@ -54,14 +56,37 @@ class VimEditor(pluginupdate.Editor): nvim_treesitter_updated = False def generate_nix( - self, plugins: List[Tuple[PluginDesc, pluginupdate.Plugin]], outfile: str + self, + plugins: List[Tuple[PluginDesc, pluginupdate.Plugin]], + outfile: str ): + log.info("Generating nix code") sorted_plugins = sorted(plugins, key=lambda v: v[0].name.lower()) + log.debug("Loading nvim-treesitter revision from nix...") nvim_treesitter_rev = pluginupdate.run_nix_expr( - "(import { }).vimPlugins.nvim-treesitter.src.rev", self.nixpkgs + "(import { }).vimPlugins.nvim-treesitter.src.rev", + self.nixpkgs, + timeout=10 ) + GET_PLUGINS_LUA = """ + with import {}; + lib.attrNames lua51Packages""" + log.debug("Loading list of lua plugins...") + luaPlugins = run_nix_expr(GET_PLUGINS_LUA, self.nixpkgs, timeout=30) + + def _isNeovimPlugin(plug: pluginupdate.Plugin) -> bool: + """ + Whether it's a neovim-only plugin + We can check if it's available in lua packages + """ + if plug.normalized_name in luaPlugins: + log.debug("%s is a neovim plugin", plug) + return True + return False + with open(outfile, "w+") as f: + log.debug("Writing to %s", outfile) f.write(HEADER) f.write( textwrap.dedent( @@ -74,7 +99,7 @@ class VimEditor(pluginupdate.Editor): ) ) for pdesc, plugin in sorted_plugins: - content = self.plugin2nix(pdesc, plugin) + content = self.plugin2nix(pdesc, plugin, _isNeovimPlugin(plugin)) f.write(content) if ( plugin.name == "nvim-treesitter" @@ -84,27 +109,10 @@ class VimEditor(pluginupdate.Editor): f.write("\n}\n") print(f"updated {outfile}") - def plugin2nix(self, pdesc: PluginDesc, plugin: pluginupdate.Plugin) -> str: - GET_PLUGINS_LUA = """ - with import {}; - lib.attrNames lua51Packages""" - luaPlugins = run_nix_expr(GET_PLUGINS_LUA, self.nixpkgs) + def plugin2nix(self, pdesc: PluginDesc, plugin: pluginupdate.Plugin, isNeovim: bool) -> str: repo = pdesc.repo - def _isNeovimPlugin(plug: pluginupdate.Plugin) -> bool: - """ - Whether it's a neovim-only plugin - We can check if it's available in lua packages - """ - # global luaPlugins - if plug.normalized_name in luaPlugins: - log.debug("%s is a neovim plugin", plug) - return True - return False - - isNeovim = _isNeovimPlugin(plugin) - content = f" {plugin.normalized_name} = " src_nix = repo.as_nix(plugin) content += """{buildFn} {{ @@ -159,8 +167,8 @@ class VimEditor(pluginupdate.Editor): def main(): global luaPlugins - log.debug(f"Loading from {ROOT}/../get-plugins.nix") - with open(f"{ROOT}/../get-plugins.nix") as f: + log.debug(f"Loading from {ROOT}/get-plugins.nix") + with open(f"{ROOT}/get-plugins.nix") as f: GET_PLUGINS = f.read() editor = VimEditor( "vim", Path("pkgs/applications/editors/vim/plugins"), GET_PLUGINS diff --git a/pkgs/applications/editors/vim/plugins/updater.nix b/pkgs/applications/editors/vim/plugins/updater.nix index afa245d9be9e..00fb8b8dedc5 100644 --- a/pkgs/applications/editors/vim/plugins/updater.nix +++ b/pkgs/applications/editors/vim/plugins/updater.nix @@ -7,14 +7,14 @@ , nurl # optional -, vimPlugins -, neovim +, neovim-unwrapped }: buildPythonApplication { - format = "other"; pname = "vim-plugins-updater"; version = "0.1"; + format = "other"; + nativeBuildInputs = [ makeWrapper python3Packages.wrapPython @@ -29,16 +29,18 @@ buildPythonApplication { installPhase = '' mkdir -p $out/bin $out/lib cp ${./update.py} $out/bin/vim-plugins-updater - cp ${./get-plugins.nix} $out/get-plugins.nix - cp ${./nvim-treesitter/update.py} $out/lib/treesitter.py - cp ${../../../../../maintainers/scripts/pluginupdate.py} $out/lib/pluginupdate.py + cp ${./get-plugins.nix} $out/bin/get-plugins.nix # wrap python scripts makeWrapperArgs+=( --prefix PATH : "${lib.makeBinPath [ - nix nix-prefetch-git neovim nurl ]}" --prefix PYTHONPATH : "$out/lib" ) + nix nix-prefetch-git neovim-unwrapped nurl ]}" --prefix PYTHONPATH : "${./.}:${../../../../../maintainers/scripts}" ) wrapPythonPrograms ''; + shellHook = '' + export PYTHONPATH=pkgs/applications/editors/vim/plugins:maintainers/scripts:$PYTHONPATH + ''; + meta.mainProgram = "vim-plugins-updater"; } diff --git a/pkgs/development/lua-modules/updater/updater.py b/pkgs/development/lua-modules/updater/updater.py index f0a91a313919..335e15c44427 100755 --- a/pkgs/development/lua-modules/updater/updater.py +++ b/pkgs/development/lua-modules/updater/updater.py @@ -177,7 +177,7 @@ def generate_pkg_nix(plug: LuaPlugin): "'version' and 'ref' will be ignored as the rockspec is hardcoded for package %s" % plug.name ) - log.warn(msg) + log.warning(msg) log.debug("Updating from rockspec %s", plug.rockspec) cmd.append(plug.rockspec)