From cb62687680f01a20c29954e502cfe408e4d4c534 Mon Sep 17 00:00:00 2001 From: Collin Funk Date: Fri, 15 Mar 2024 19:58:27 -0700 Subject: [PATCH] gnulib-tool.py: Follow gnulib-tool changes, part 58. Follow gnulib-tool change 2017-05-21 Bruno Haible gnulib-tool: Add options to create hard links. * pygnulib/GLConfig.py (GLConfig.__init__): Add 'copymode' and 'lcopymode' to the parameter list. Initialize them. (GLConfig.default): Don't use symbolic or hard links by default. (GLConfig.checkCopyMode, GLConfig.setCopyMode, GLConfig.resetCopyMode): New functions to modify and check the method for copying non --local-dir files. (GLConfig.checkLCopyMode, GLConfig.setLCopyMode) (GLConfig.resetLCopyMode): New functions to modify and check the method for copying --local-dir files. (GLConfig.checkSymbolic, GLConfig.resetSymbolic, GLConfig.setSymbolic) (GLConfig.checkLSymbolic, GLConfig.resetLSymbolic) (GLConfig.setLSymbolic): Remove unused functions. The functionality of these are now implemented in the *CopyMode() and *LCopyMode() variants. * pygnulib/GLFileSystem.py (CopyAction.Hardlink): New Enum value to describe hard links. (GLFileSystem.shouldLink): Check if hard links should be used. (GLFileAssistant.add, GLFileAssistant.update): Try to hard link if enabled. Copy the file if linking fails. (GLFileAssistant.add_or_update): Remove temporary files unconditionally. * pygnulib/GLInfo.py (GLInfo.usage): Document new options in the usage message. * pygnulib/GLTestDir.py (GLTestDir.execute): Try to hard link if enabled. Copy the file if linking fails. * pygnulib/classes.py: Importy the CopyAction Enum. * pygnulib/constants.py (hardlink): New function based on symlink_relative. * pygnulib/main.py (main): Add new options --hardlink and --local-hardlink. Invoke 'git update-index --refresh' to mitigate the effects of the hard links on git. --- ChangeLog | 37 ++++++++++++++++ gnulib-tool.py.TODO | 31 +------------ pygnulib/GLConfig.py | 94 +++++++++++++++++++++------------------- pygnulib/GLFileSystem.py | 63 +++++++++++++++++---------- pygnulib/GLInfo.py | 4 ++ pygnulib/GLTestDir.py | 4 +- pygnulib/classes.py | 4 +- pygnulib/constants.py | 20 +++++++++ pygnulib/main.py | 41 ++++++++++++++---- 9 files changed, 191 insertions(+), 107 deletions(-) diff --git a/ChangeLog b/ChangeLog index 269fe5f5ff..4f2e60f336 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,40 @@ +2024-03-15 Collin Funk + + + gnulib-tool.py: Follow gnulib-tool changes, part 58. + Follow gnulib-tool change + 2017-05-21 Bruno Haible + gnulib-tool: Add options to create hard links. + * pygnulib/GLConfig.py (GLConfig.__init__): Add 'copymode' and + 'lcopymode' to the parameter list. Initialize them. + (GLConfig.default): Don't use symbolic or hard links by default. + (GLConfig.checkCopyMode, GLConfig.setCopyMode, GLConfig.resetCopyMode): + New functions to modify and check the method for copying non --local-dir + files. + (GLConfig.checkLCopyMode, GLConfig.setLCopyMode) + (GLConfig.resetLCopyMode): New functions to modify and check the method + for copying --local-dir files. + (GLConfig.checkSymbolic, GLConfig.resetSymbolic, GLConfig.setSymbolic) + (GLConfig.checkLSymbolic, GLConfig.resetLSymbolic) + (GLConfig.setLSymbolic): Remove unused functions. The functionality of + these are now implemented in the *CopyMode() and *LCopyMode() variants. + * pygnulib/GLFileSystem.py (CopyAction.Hardlink): New Enum value to + describe hard links. + (GLFileSystem.shouldLink): Check if hard links should be used. + (GLFileAssistant.add, GLFileAssistant.update): Try to hard link if + enabled. Copy the file if linking fails. + (GLFileAssistant.add_or_update): Remove temporary files unconditionally. + * pygnulib/GLInfo.py (GLInfo.usage): Document new options in the usage + message. + * pygnulib/GLTestDir.py (GLTestDir.execute): Try to hard link if + enabled. Copy the file if linking fails. + * pygnulib/classes.py: Importy the CopyAction Enum. + * pygnulib/constants.py (hardlink): New function based on + symlink_relative. + * pygnulib/main.py (main): Add new options --hardlink and + --local-hardlink. Invoke 'git update-index --refresh' to mitigate the + effects of the hard links on git. + 2024-03-16 Bruno Haible gnulib-tool.sh: Make --local-dir options in gnulib-cache.m4 look nicer. diff --git a/gnulib-tool.py.TODO b/gnulib-tool.py.TODO index b5796b24a3..eb315de832 100644 --- a/gnulib-tool.py.TODO +++ b/gnulib-tool.py.TODO @@ -16,10 +16,7 @@ The following commits to gnulib-tool have not yet been reflected in -------------------------------------------------------------------------------- Implement the options: - -h | --hardlink - --local-hardlink - -S | --more-symlinks - -H | --more-hardlinks + --automake-subdir-tests --help (same output) Optimize: @@ -198,32 +195,6 @@ Date: Thu Jun 8 15:09:31 2017 +0200 -------------------------------------------------------------------------------- -commit 306be564ba47ec412ca158f66ffa90a058f5253b -Author: Bruno Haible -Date: Mon May 22 01:39:59 2017 +0200 - - gnulib-tool: Add options to create hard links. - - * gnulib-tool (func_usage): Document options --hardlink, - --local-hardlink, --more-hardlinks. - (func_symlink): Renamed from func_ln. - (func_symlink_if_changed): Renamed from func_ln_if_changed. - (func_hardlink): New function. - (copymode, lcopymode): New variables. - (symbolic, lsymbolic): Remove variables. - (Options): Implement options --hardlink, --local-hardlink, - --more-hardlinks. - (func_should_link): Renamed from func_should_symlink. Set copyaction. - (func_add_file, func_update_file): Update invocation of - func_should_link. Invoke func_hardlink when appropriate. - (func_import): Update comments. - (func_create_testdir): Update invocation of func_should_link. Invoke - func_hardlink when appropriate. - Finally, invoke 'git update-index --refresh' to mitigate the effects of - the hard links on git. - --------------------------------------------------------------------------------- - commit f5142421c62024efa22cd4429100c4d9c1cc2ac4 Author: Bruno Haible Date: Sat May 20 13:24:37 2017 +0200 diff --git a/pygnulib/GLConfig.py b/pygnulib/GLConfig.py index 049fe35a2c..ecfbb13353 100644 --- a/pygnulib/GLConfig.py +++ b/pygnulib/GLConfig.py @@ -21,6 +21,7 @@ import copy import tempfile from . import constants from .GLError import GLError +from pygnulib import classes #=============================================================================== @@ -58,10 +59,9 @@ class GLConfig(object): incl_test_categories=None, excl_test_categories=None, libname=None, lgpl=None, gnu_make=None, makefile_name=None, tests_makefile_name=None, automake_subdir=None, libtool=None, conddeps=None, macro_prefix=None, - podomain=None, witness_c_macro=None, vc_files=None, symbolic=None, - lsymbolic=None, configure_ac=None, ac_version=None, - libtests=None, single_configure=None, verbose=None, dryrun=None, - errors=None): + podomain=None, witness_c_macro=None, vc_files=None, copymode=None, + lcopymode=None, configure_ac=None, ac_version=None, libtests=None, + single_configure=None, verbose=None, dryrun=None, errors=None): '''GLConfig.__init__(arguments) -> GLConfig Create new GLConfig instance.''' @@ -171,14 +171,18 @@ class GLConfig(object): self.resetVCFiles() if vc_files != None: self.setVCFiles(vc_files) - # symbolic - self.resetSymbolic() - if symbolic != None: - self.setSymbolic(symbolic) - # lsymbolic - self.resetLSymbolic() - if lsymbolic != None: - self.setLSymbolic(lsymbolic) + # copymode (--symlink and --hardlink) + self.resetCopyMode() + if copymode == None: + # Default to copying. + copymode = classes.CopyAction.Copy + self.setCopyMode(copymode) + # lcopymode (--local-symlink and --local-hardlink) + self.resetLCopyMode() + if lcopymode == None: + # Default to copying. + lcopymode = classes.CopyAction.Copy + self.setLCopyMode(lcopymode) # configure_ac self.resetAutoconfFile() if configure_ac != None: @@ -285,9 +289,11 @@ class GLConfig(object): elif key in ['localpath', 'modules', 'avoids', 'tests', 'incl_test_categories', 'excl_test_categories']: return list() - elif key in ['libtool', 'gnu_make', 'automake_subdir', 'conddeps', 'symbolic', - 'lsymbolic', 'libtests', 'dryrun']: + elif key in ['libtool', 'gnu_make', 'automake_subdir', 'conddeps', + 'libtests', 'dryrun']: return False + elif key in ['copymode', 'lcopymode']: + return classes.CopyAction.Copy elif key in ['lgpl', 'vc_files']: return None elif key == 'errors': @@ -1007,42 +1013,42 @@ class GLConfig(object): '''Specify preferred autoconf version. Default value is 2.64.''' self.table['ac_version'] = 2.64 - # Define symbolic methods. - def checkSymbolic(self): - '''Check if pygnulib will make symbolic links instead of copying files.''' - return self.table['symbolic'] - - def setSymbolic(self, value): - '''Enable / disable creation of the symbolic links instead of copying files.''' - if type(value) is bool: - self.table['symbolic'] = value - else: # if type(value) is not bool - raise TypeError('value must be a bool, not %s' + # Define copymode methods. + def checkCopyMode(self): + '''Check if pygnulib will copy files, create symlinks, or create hard links.''' + return self.table['copymode'] + + def setCopyMode(self, value): + '''Change the method used for copying / linking files.''' + if type(value) is classes.CopyAction: + self.table['copymode'] = value + else: # if type(value) is not CopyAction + raise TypeError('value must be a CopyAction, not %s' % type(value).__name__) - def resetSymbolic(self): - '''Reset creation of the symbolic links instead of copying files.''' - self.table['symbolic'] = False - - # Define lsymbolic methods. - def checkLSymbolic(self): - '''Check if pygnulib will make symbolic links instead of copying files, - only for files from the local override directories.''' - return self.table['lsymbolic'] + def resetCopyMode(self): + '''Reset the method used for creating files to copying instead of linking.''' + self.table['copymode'] = classes.CopyAction.Copy - def setLSymbolic(self, value): - '''Enable / disable creation of symbolic links instead of copying files, + # Define lcopymode methods. + def checkLCopyMode(self): + '''Check if pygnulib will copy files, create symlinks, or create hard links, only for files from the local override directories.''' - if type(value) is bool: - self.table['lsymbolic'] = value - else: # if type(value) is not bool - raise TypeError('value must be a bool, not %s' + return self.table['lcopymode'] + + def setLCopyMode(self, value): + '''Change the method used for copying / linking files, only for files from + the local override directories.''' + if type(value) is classes.CopyAction: + self.table['lcopymode'] = value + else: # if type(value) is not CopyAction + raise TypeError('value must be a CopyAction, not %s' % type(value).__name__) - def resetLSymbolic(self): - '''Reset creation of symbolic links instead of copying files, only for - files from the local override directories.''' - self.table['lsymbolic'] = False + def resetLCopyMode(self): + '''Reset the method used for creating files to copying instead of linking, + only for files from the local override directories.''' + self.table['lcopymode'] = classes.CopyAction.Copy # Define verbosity methods. def getVerbosity(self): diff --git a/pygnulib/GLFileSystem.py b/pygnulib/GLFileSystem.py index f60e3fc6ea..7c6c9a15ef 100644 --- a/pygnulib/GLFileSystem.py +++ b/pygnulib/GLFileSystem.py @@ -38,11 +38,15 @@ __copyright__ = constants.__copyright__ # Define global constants #=============================================================================== DIRS = constants.DIRS +substart = constants.substart joinpath = constants.joinpath copyfile = constants.copyfile movefile = constants.movefile +hardlink = constants.hardlink +link_if_changed = constants.link_if_changed isdir = os.path.isdir isfile = os.path.isfile +islink = os.path.islink #=============================================================================== @@ -51,6 +55,7 @@ isfile = os.path.isfile class CopyAction(Enum): Copy = 0 Symlink = 1 + Hardlink = 2 #=============================================================================== @@ -134,18 +139,20 @@ class GLFileSystem(object): def shouldLink(self, original, lookedup): '''GLFileSystem.shouldLink(original, lookedup) - Determines whether the original file should be copied or symlinked. + Determines whether the original file should be copied, symlinked, + or hardlinked. Returns a CopyAction.''' - symbolic = self.config['symbolic'] - lsymbolic = self.config['lsymbolic'] + copymode = self.config['copymode'] + lcopymode = self.config['lcopymode'] localpath = self.config['localpath'] - if symbolic: - return CopyAction.Symlink - if lsymbolic: + + # Don't bother checking the localpath's if we end up performing the same + # action anyways. + if copymode != lcopymode: for localdir in localpath: if lookedup == joinpath(localdir, original): - return CopyAction.Symlink - return CopyAction.Copy + return lcopymode + return copymode #=============================================================================== @@ -259,12 +266,16 @@ class GLFileAssistant(object): print('Copying file %s' % rewritten) if self.filesystem.shouldLink(original, lookedup) == CopyAction.Symlink \ and not tmpflag and filecmp.cmp(lookedup, tmpfile): - constants.link_if_changed(lookedup, joinpath(destdir, rewritten)) + link_if_changed(lookedup, joinpath(destdir, rewritten)) else: # if any of these conditions is not met - try: # Try to move file - movefile(tmpfile, joinpath(destdir, rewritten)) - except Exception as error: - raise GLError(17, original) + if self.filesystem.shouldLink(original, lookedup) == CopyAction.Hardlink \ + and not tmpflag and filecmp.cmp(lookedup, tmpfile): + hardlink(lookedup, joinpath(destdir, rewritten)) + else: # Move instead of linking. + try: # Try to move file + movefile(tmpfile, joinpath(destdir, rewritten)) + except Exception as error: + raise GLError(17, original) else: # if self.config['dryrun'] print('Copy file %s' % rewritten) @@ -308,14 +319,18 @@ class GLFileAssistant(object): raise GLError(17, original) if self.filesystem.shouldLink(original, lookedup) == CopyAction.Symlink \ and not tmpflag and filecmp.cmp(lookedup, tmpfile): - constants.link_if_changed(lookedup, basepath) + link_if_changed(lookedup, basepath) else: # if any of these conditions is not met - try: # Try to move file - if os.path.exists(basepath): - os.remove(basepath) - copyfile(tmpfile, rewritten) - except Exception as error: - raise GLError(17, original) + if self.filesystem.shouldLink(original, lookedup) == CopyAction.Hardlink \ + and not tmpflag and filecmp.cmp(lookedup, tmpfile): + hardlink(lookedup, basepath) + else: # Move instead of linking. + try: # Try to move file + if os.path.exists(basepath): + os.remove(basepath) + copyfile(tmpfile, joinpath(destdir, rewritten)) + except Exception as error: + raise GLError(17, original) else: # if self.config['dryrun'] if already_present: print('Update file %s (backup in %s)' % (rewritten, backupname)) @@ -337,7 +352,7 @@ class GLFileAssistant(object): % type(already_present).__name__) xoriginal = original if original.startswith('tests=lib/'): - xoriginal = constants.substart('tests=lib/', 'lib/', original) + xoriginal = substart('tests=lib/', 'lib/', original) lookedup, tmpflag = self.filesystem.lookup(xoriginal) tmpfile = self.tmpfilename(rewritten) sed_transform_lib_file = self.transformers.get('lib', '') @@ -372,11 +387,15 @@ class GLFileAssistant(object): file.write(data) path = joinpath(self.config['destdir'], rewritten) if isfile(path): + # The file already exists. self.update(lookedup, tmpflag, tmpfile, already_present) - os.remove(tmpfile) else: # if not isfile(path) + # Install the file. + # Don't protest if the file should be there but isn't: it happens + # frequently that developers don't put autogenerated files under version control. self.add(lookedup, tmpflag, tmpfile) self.addFile(rewritten) + os.remove(tmpfile) def super_update(self, basename, tmpfile): '''GLFileAssistant.super_update(basename, tmpfile) -> tuple diff --git a/pygnulib/GLInfo.py b/pygnulib/GLInfo.py index cb77c5e923..8851341c4d 100644 --- a/pygnulib/GLInfo.py +++ b/pygnulib/GLInfo.py @@ -315,10 +315,14 @@ Options for --import, --add/remove-import, --update, -s, --symbolic, --symlink Make symbolic links instead of copying files. --local-symlink Make symbolic links instead of copying files, only for files from the local override directory. + -h, --hardlink Make hard links instead of copying files. + --local-hardlink Make hard links instead of copying files, only + for files from the local override directory. Options for --import, --add/remove-import, --update: -S, --more-symlinks Deprecated; equivalent to --symlink. + -H, --more-hardlinks Deprecated; equivalent to --hardlink. Report bugs to .''' return result diff --git a/pygnulib/GLTestDir.py b/pygnulib/GLTestDir.py index f747a480f5..024260f655 100644 --- a/pygnulib/GLTestDir.py +++ b/pygnulib/GLTestDir.py @@ -344,7 +344,7 @@ class GLTestDir(object): for file in self.rewrite_files(filelist)] directories = sorted(set(directories)) - # Copy files or make symbolic links. + # Copy files or make symbolic links or hard links. filetable = list() for src in filelist: dest = self.rewrite_files([src])[-1] @@ -366,6 +366,8 @@ class GLTestDir(object): else: # if not flag if self.filesystem.shouldLink(src, lookedup) == CopyAction.Symlink: constants.link_relative(lookedup, destpath) + elif self.filesystem.shouldLink(src, lookedup) == CopyAction.Hardlink: + constants.hardlink(lookedup, destpath) else: copyfile(lookedup, destpath) diff --git a/pygnulib/classes.py b/pygnulib/classes.py index 088e4e5d75..537872a37b 100644 --- a/pygnulib/classes.py +++ b/pygnulib/classes.py @@ -30,6 +30,7 @@ try: from .GLInfo import GLInfo # File system + from .GLFileSystem import CopyAction from .GLFileSystem import GLFileSystem from .GLFileSystem import GLFileAssistant @@ -56,6 +57,7 @@ except ValueError as error: from GLInfo import GLInfo # File system + from GLFileSystem import CopyAction from GLFileSystem import GLFileSystem from GLFileSystem import GLFileAssistant @@ -75,7 +77,7 @@ except ValueError as error: # Append modules to namespace. __all__ += ['GLConfig', 'GLError', 'GLInfo'] -__all__ += ['GLFileSystem', 'GLFileAssistant'] +__all__ += ['CopyAction', 'GLFileSystem', 'GLFileAssistant'] __all__ += ['GLModule', 'GLModuleSystem', 'GLModuleTable'] __all__ += ['GLImport', 'GLEmiter', 'GLTestDir'] __all__ += ['GLMakefileTable'] diff --git a/pygnulib/constants.py b/pygnulib/constants.py index a3bf8cd1d4..dc2e68069d 100644 --- a/pygnulib/constants.py +++ b/pygnulib/constants.py @@ -412,6 +412,26 @@ def link_if_changed(src, dest): symlink_relative(link_value, dest) +def hardlink(src: str, dest: str) -> None: + '''Like ln, except use cp -p if ln fails. + src is either absolute or relative to the directory of dest.''' + try: + os.link(src, dest) + except PermissionError: + sys.stderr.write('%s: ln failed; falling back on cp -p\n' % APP['name']) + if src.startswith('/') or (len(src) >= 2 and src[1] == ':'): + # src is absolute. + cp_src = src + else: + # src is relative to the directory of dest. + last_slash = dest.rfind('/') + if last_slash >= 0: + cp_src = joinpath(dest[0: last_slash - 1], src) + else: + cp_src = src + copyfile2(cp_src, dest) + + def filter_filelist(separator, filelist, prefix, suffix, removed_prefix, removed_suffix, added_prefix='', added_suffix=''): diff --git a/pygnulib/main.py b/pygnulib/main.py index 5d65a45b25..a3db7bb7c7 100644 --- a/pygnulib/main.py +++ b/pygnulib/main.py @@ -436,15 +436,25 @@ def main(): default=None, action='store_true') # symlink - parser.add_argument('-s', '--symbolic', '--symlink', - dest='symlink', + parser.add_argument('-s', '-S', '--symbolic', '--symlink', '--more-symlinks', + dest='copymode', default=None, - action='store_true') + action='store_const', const=classes.CopyAction.Symlink) # local-symlink parser.add_argument('--local-symlink', - dest='lsymlink', + dest='lcopymode', default=None, - action='store_true') + action='store_const', const=classes.CopyAction.Symlink) + # hardlink + parser.add_argument('-h', '-H', '--hardlink', '--more-hardlinks', + dest='copymode', + default=None, + action='store_const', const=classes.CopyAction.Hardlink) + # local-hardlink + parser.add_argument('--local-hardlink', + dest='lcopymode', + default=None, + action='store_const', const=classes.CopyAction.Hardlink) # All other arguments are collected. parser.add_argument("non_option_arguments", nargs='*') @@ -748,8 +758,8 @@ def main(): avoids = [ module for list1 in avoids for module in list1 ] - symlink = cmdargs.symlink == True - lsymlink = cmdargs.lsymlink == True + copymode = cmdargs.copymode + lcopymode = cmdargs.lcopymode single_configure = cmdargs.single_configure docbase = None @@ -779,8 +789,8 @@ def main(): podomain=podomain, witness_c_macro=witness_c_macro, vc_files=vc_files, - symbolic=symlink, - lsymbolic=lsymlink, + copymode=copymode, + lcopymode=lcopymode, single_configure=single_configure, verbose=verbose, dryrun=dryrun, @@ -1269,6 +1279,19 @@ def main(): sys.stderr.write(message) sys.exit(1) + if copymode != classes.CopyAction.Copy or lcopymode != classes.CopyAction.Copy: + # Setting hard links modifies the ctime of files in the gnulib checkout. + # This disturbs the result of the next "gitk" invocation. + # Workaround: Let git scan the files. This can be done through + # "git update-index --refresh" or "git status" or "git diff". + if isdir(joinpath(APP['root'], '.git')): + try: + sp.run(['git', 'update-index', '--refresh'], + cwd=APP['root'], stdout=sp.DEVNULL, stderr=sp.DEVNULL) + except Exception: + # We did our best... + pass + if __name__ == '__main__': try: # Try to execute -- 2.39.5