diff --git a/NEWS b/NEWS index 4ba2231..db0cbd4 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,41 @@ +0.151 + - fixed shell command injection via crafted _service files (CVE-2015-0778) + - fix times when data comes from OBS backend + - support updateing the link in target package for submit requests + - various minor bugfixes + +0.150 + - support local builds using builenv (for same build environment as a former build) + - add "osc api --edit" option to be able to edit some meta files directly + - follow the request order of the api (sorting according to priorization) + - add mr --release-project option for kgraft updates + - add support for makeoriginolder in request + +0.149 + - removed "--diff" option from the "createrequest" command + - introduced new "vc-cmd" config option, which is used to specify the path + to the vc script + - various bugfixes + +0.148 + - support new history including review history of OBS 2.6 + - display request priorities, if important or critical + - add "osc rq priorize" command to re-priorize existing requests + - allow also "osc rq ls" shortcut + - fish shell completion support + +0.147 + - support groups in maintainership requests + - fixing listing of review requests + - support expanded package listing (when using project links) + - fixing "osc add git://" behaviour + - using xz as default compression + - support local debian live (image) build format + - handle ppc64le for debian as well + - fix buildlog --strip-time + - some more minor bugfixes + - speedup update of a project working copy (in some cases) + 0.146 - support maintenance release request with acceptinfo data (OBS 2.6) - setlinkrev can be used to update frozen links to current revisions again diff --git a/osc.fish b/osc.fish new file mode 100644 index 0000000..09dac9d --- /dev/null +++ b/osc.fish @@ -0,0 +1,116 @@ +# fish completion for git +# vim: smartindent:expandtab:ts=2:sw=2 + +function __fish_osc_needs_command + set cmd (commandline -opc) + if contains "$cmd" 'osc' 'osc help' + return 0 + end + return 1 +end + +function __fish_osc_using_command + set cmd (commandline -opc) + if [ (count $cmd) -gt 1 ] + for arg in $argv + if [ $arg = $cmd[2] ] + return 0 + end + end + end + return 1 +end + +# general options +complete -f -c osc -n 'not __fish_osc_needs_command' -s A -l apiurl -d 'specify URL to access API server at or an alias' +complete -f -c osc -n 'not __fish_osc_needs_command' -s c -l config -d 'specify alternate configuration file' +complete -f -c osc -n 'not __fish_osc_needs_command' -s d -l debug -d 'print info useful for debugging' +complete -f -c osc -n 'not __fish_osc_needs_command' -l debugger -d 'jump into the debugger before executing anything' +complete -f -c osc -n 'not __fish_osc_needs_command' -s h -l help -d 'show this help message and exit' +complete -f -c osc -n 'not __fish_osc_needs_command' -s H -l http-debug -d 'debug HTTP traffic (filters some headers)' +complete -f -c osc -n 'not __fish_osc_needs_command' -l http-full-debug -d 'debug HTTP traffic (filters no headers)' +complete -f -c osc -n 'not __fish_osc_needs_command' -l no-gnome-keyring -d 'disable usage of GNOME Keyring' +complete -f -c osc -n 'not __fish_osc_needs_command' -l no-keyring -d 'disable usage of desktop keyring system' +complete -f -c osc -n 'not __fish_osc_needs_command' -l post-mortem -d 'jump into the debugger in case of errors' +complete -f -c osc -n 'not __fish_osc_needs_command' -s q -l quiet -d 'be quiet, not verbose' +complete -f -c osc -n 'not __fish_osc_needs_command' -s t -l traceback -d 'print call trace in case of errors' +complete -f -c osc -n 'not __fish_osc_needs_command' -s v -l verbose -d 'increase verbosity' +complete -f -c osc -n 'not __fish_osc_needs_command' -l version -d 'show program\'s version number and exit' + +# osc commands +complete -f -c osc -n '__fish_osc_needs_command' -a 'add' -d 'Mark files to be added upon the next commit' +complete -f -c osc -n '__fish_osc_needs_command' -a 'addremove ar' -d 'Adds new files, removes disappeared files' +complete -f -c osc -n '__fish_osc_needs_command' -a 'aggregatepac' -d '"Aggregate" a package to another package' +complete -f -c osc -n '__fish_osc_needs_command' -a 'api' -d 'Issue an arbitrary request to the API' +complete -f -c osc -n '__fish_osc_needs_command' -a 'branch bco branchco getpac' -d 'Branch a package' +complete -f -c osc -n '__fish_osc_needs_command' -a 'chroot' -d 'into the buildchroot' +complete -f -c osc -n '__fish_osc_needs_command' -a 'clean' -d 'removes all untracked files from the package working ...' +complete -f -c osc -n '__fish_osc_needs_command' -a 'commit checkin ci' -d 'Upload content to the repository server' +complete -f -c osc -n '__fish_osc_needs_command' -a 'config' -d 'get/set a config option' +complete -f -c osc -n '__fish_osc_needs_command' -a 'copypac' -d 'Copy a package' +complete -f -c osc -n '__fish_osc_needs_command' -a 'createincident' -d 'Create a maintenance incident' +complete -f -c osc -n '__fish_osc_needs_command' -a 'createrequest creq' -d 'create multiple requests with a single command' +complete -f -c osc -n '__fish_osc_needs_command' -a 'delete del remove rm' -d 'Mark files or package directories to be deleted upon ...' +complete -f -c osc -n '__fish_osc_needs_command' -a 'deleterequest deletereq dr dropreq droprequest' -d 'Request to delete (or "drop") a package or project' +complete -f -c osc -n '__fish_osc_needs_command' -a 'dependson whatdependson' -d 'Show the build dependencies' +complete -f -c osc -n '__fish_osc_needs_command' -a 'detachbranch' -d 'replace a link with its expanded sources' +complete -f -c osc -n '__fish_osc_needs_command' -a 'develproject bsdevelproject dp' -d 'print the devel project / package of a package' +complete -f -c osc -n '__fish_osc_needs_command' -a 'diff di ldiff linkdiff' -d 'Generates a diff' +complete -f -c osc -n '__fish_osc_needs_command' -a 'distributions dists' -d 'Shows all available distributions' +complete -f -c osc -n '__fish_osc_needs_command' -a 'getbinaries' -d 'Download binaries to a local directory' +complete -f -c osc -n '__fish_osc_needs_command' -a 'help ? h' -d 'give detailed help on a specific sub-command' +complete -f -c osc -n '__fish_osc_needs_command' -a 'importsrcpkg' -d 'Import a new package from a src.rpm' +complete -f -c osc -n '__fish_osc_needs_command' -a 'info' -d 'Print information about a working copy' +complete -f -c osc -n '__fish_osc_needs_command' -a 'init' -d 'Initialize a directory as working copy' +complete -f -c osc -n '__fish_osc_needs_command' -a 'jobhistory jobhist' -d 'Shows the job history of a project' +complete -f -c osc -n '__fish_osc_needs_command' -a 'linkpac' -d '"Link" a package to another package' +complete -f -c osc -n '__fish_osc_needs_command' -a 'linktobranch' -d 'Convert a package containing a classic link with patc...' +complete -f -c osc -n '__fish_osc_needs_command' -a 'list LL lL ll ls' -d 'List sources or binaries on the server' +complete -f -c osc -n '__fish_osc_needs_command' -a 'localbuildlog lbl' -d 'Shows the build log of a local buildchroot' +complete -f -c osc -n '__fish_osc_needs_command' -a 'log' -d 'Shows the commit log of a package' +complete -f -c osc -n '__fish_osc_needs_command' -a 'maintainer bugowner' -d 'Show maintainers according to server side configuration' +complete -f -c osc -n '__fish_osc_needs_command' -a 'maintenancerequest mr' -d 'Create a request for starting a maintenance incident.' +complete -f -c osc -n '__fish_osc_needs_command' -a 'man' -d 'generates a man page' +complete -f -c osc -n '__fish_osc_needs_command' -a 'mbranch maintained sm' -d 'Search or banch multiple instances of a package' +complete -f -c osc -n '__fish_osc_needs_command' -a 'meta' -d 'Show meta information, or edit it' +complete -f -c osc -n '__fish_osc_needs_command' -a 'mkpac' -d 'Create a new package under version control' +complete -f -c osc -n '__fish_osc_needs_command' -a 'mv' -d 'Move SOURCE file to DEST and keep it under version co...' +complete -f -c osc -n '__fish_osc_needs_command' -a 'my' -d 'show waiting work, packages, projects or requests inv...' +complete -f -c osc -n '__fish_osc_needs_command' -a 'patchinfo' -d 'Generate and edit a patchinfo file.' +complete -f -c osc -n '__fish_osc_needs_command' -a 'pdiff' -d 'Quick alias to diff the content of a package with its...' +complete -f -c osc -n '__fish_osc_needs_command' -a 'prdiff projdiff projectdiff' -d 'Server-side diff of two projects' +complete -f -c osc -n '__fish_osc_needs_command' -a 'prjresults pr' -d 'Shows project-wide build results' +complete -f -c osc -n '__fish_osc_needs_command' -a 'pull' -d 'merge the changes of the link target into your workin...' +complete -f -c osc -n '__fish_osc_needs_command' -a 'rdelete' -d 'Delete a project or packages on the server.' +complete -f -c osc -n '__fish_osc_needs_command' -a 'rdiff' -d 'Server-side "pretty" diff of two packages' +complete -f -c osc -n '__fish_osc_needs_command' -a 'rebuild rebuildpac' -d 'Trigger package rebuilds' +complete -f -c osc -n '__fish_osc_needs_command' -a 'release' -d 'Release sources and binaries' +complete -f -c osc -n '__fish_osc_needs_command' -a 'releaserequest' -d 'Create a request for releasing a maintenance update.' +complete -f -c osc -n '__fish_osc_needs_command' -a 'remotebuildlog rbl rblt rbuildlog rbuildlogtail remotebuildlogtail' -d 'Shows the build log of a package' +complete -f -c osc -n '__fish_osc_needs_command' -a 'repairlink' -d 'Repair a broken source link' +complete -f -c osc -n '__fish_osc_needs_command' -a 'repairwc' -d 'try to repair an inconsistent working copy' +complete -f -c osc -n '__fish_osc_needs_command' -a 'repositories platforms repos' -d 'shows repositories configured for a project. It skips...' +complete -f -c osc -n '__fish_osc_needs_command' -a 'repourls' -d 'Shows URLs of .repo files' +complete -f -c osc -n '__fish_osc_needs_command' -a 'request review rq' -d 'Show or modify requests and reviews' +complete -f -c osc -n '__fish_osc_needs_command' -a 'requestmaintainership reqbs reqbugownership reqmaintainership reqms requestbugownership' -d 'requests to add user as maintainer or bugowner' +complete -f -c osc -n '__fish_osc_needs_command' -a 'resolved' -d 'Remove "conflicted" state on working copy files' +complete -f -c osc -n '__fish_osc_needs_command' -a 'restartbuild abortbuild' -d 'Restart the build of a certain project or package' +complete -f -c osc -n '__fish_osc_needs_command' -a 'results r' -d 'Shows the build results of a package or project' +complete -f -c osc -n '__fish_osc_needs_command' -a 'revert' -d 'Restore changed files or the entire working copy.' +complete -f -c osc -n '__fish_osc_needs_command' -a 'rremove' -d 'Remove source files from selected package' +complete -f -c osc -n '__fish_osc_needs_command' -a 'search bse se' -d 'Search for a project and/or package.' +complete -f -c osc -n '__fish_osc_needs_command' -a 'service' -d 'Handle source services' +complete -f -c osc -n '__fish_osc_needs_command' -a 'setdevelproject sdp' -d 'Set the devel project / package of a package' +complete -f -c osc -n '__fish_osc_needs_command' -a 'setlinkrev' -d 'Updates a revision number in a source link.' +complete -f -c osc -n '__fish_osc_needs_command' -a 'signkey' -d 'Manage Project Signing Key' +complete -f -c osc -n '__fish_osc_needs_command' -a 'status st' -d 'Show status of files in working copy' +complete -f -c osc -n '__fish_osc_needs_command' -a 'submitrequest sr submitpac submitreq' -d 'Create request to submit source into another Project' +complete -f -c osc -n '__fish_osc_needs_command' -a 'token' -d 'Show and manage authentication token' +complete -f -c osc -n '__fish_osc_needs_command' -a 'triggerreason tr' -d 'Show reason why a package got triggered to build' +complete -f -c osc -n '__fish_osc_needs_command' -a 'undelete' -d 'Restores a deleted project or package on the server.' +complete -f -c osc -n '__fish_osc_needs_command' -a 'unlock' -d 'Unlocks a project or package' +complete -f -c osc -n '__fish_osc_needs_command' -a 'update up' -d 'Update a working copy' +complete -f -c osc -n '__fish_osc_needs_command' -a 'updatepacmetafromspec metafromspec updatepkgmetafromspec' -d 'Update package meta information from a specfile' +complete -f -c osc -n '__fish_osc_needs_command' -a 'vc' -d 'Edit the changes file' +complete -f -c osc -n '__fish_osc_needs_command' -a 'whois user who' -d 'Show fullname and email of a buildservice user' +complete -f -c osc -n '__fish_osc_needs_command' -a 'wipebinaries' -d 'Delete all binary packages of a certain project/package' diff --git a/osc/OscConfigParser.py b/osc/OscConfigParser.py index 3ed954f..bf154b1 100644 --- a/osc/OscConfigParser.py +++ b/osc/OscConfigParser.py @@ -323,6 +323,16 @@ def write(self, fp, comments = False): else: configparser.SafeConfigParser.write(self, fp) + def has_option(self, section, option, proper=False, **kwargs): + """ + Returns True, if the passed section contains the specified option. + If proper is True, True is only returned if the option is owned by + this section and not "inherited" from the default. + """ + if proper: + return self.optionxform(option) in self._sections[section].keys() + return configparser.SafeConfigParser.has_option(self, section, option, **kwargs) + # XXX: simplify! def __str__(self): ret = [] diff --git a/osc/babysitter.py b/osc/babysitter.py index 544418c..682de94 100644 --- a/osc/babysitter.py +++ b/osc/babysitter.py @@ -11,6 +11,7 @@ import sys import signal import traceback +from urlgrabber.grabber import URLGrabError from osc import oscerr from .oscsslexcp import NoSecureSSLError @@ -140,6 +141,9 @@ def run(prg, argv=None): except URLError as e: print('Failed to reach a server:\n', e.reason, file=sys.stderr) return 1 + except URLGrabError as e: + print('Failed to grab %s: %s' % (e.url, e.exception), file=sys.stderr) + return 1 except IOError as e: # ignore broken pipe if e.errno != errno.EPIPE: diff --git a/osc/build.py b/osc/build.py index 76d4eb9..71d71a5 100644 --- a/osc/build.py +++ b/osc/build.py @@ -53,20 +53,20 @@ ] can_also_build = { - 'aarch64':['aarch64'], # only needed due to used heuristics in build parameter evaluation - 'armv6l' :[ 'armv4l', 'armv5l', 'armv6l', 'armv5el', 'armv6el' ], - 'armv7l' :[ 'armv4l', 'armv5l', 'armv6l', 'armv7l', 'armv5el', 'armv6el', 'armv7el' ], - 'armv5el':[ 'armv4l', 'armv5l', 'armv5el' ], # not existing arch, just for compatibility - 'armv6el':[ 'armv4l', 'armv5l', 'armv6l', 'armv5el', 'armv6el' ], # not existing arch, just for compatibility - 'armv6hl':[ 'armv4l', 'armv5l', 'armv6l', 'armv5el', 'armv6el' ], - 'armv7el':[ 'armv4l', 'armv5l', 'armv6l', 'armv7l', 'armv5el', 'armv6el', 'armv7el' ], # not existing arch, just for compatibility - 'armv7hl':[ 'armv7hl' ], # not existing arch, just for compatibility - 'armv8el':[ 'armv4l', 'armv5el', 'armv6el', 'armv7el', 'armv8el' ], # not existing arch, just for compatibility - 'armv8l' :[ 'armv4l', 'armv5el', 'armv6el', 'armv7el', 'armv8el' ], # not existing arch, just for compatibility - 'armv5tel':[ 'armv4l', 'armv5el', 'armv5tel' ], + 'aarch64': ['aarch64'], # only needed due to used heuristics in build parameter evaluation + 'armv6l': [ 'armv4l', 'armv5l', 'armv6l', 'armv5el', 'armv6el' ], + 'armv7l': [ 'armv4l', 'armv5l', 'armv6l', 'armv7l', 'armv5el', 'armv6el', 'armv7el' ], + 'armv5el': [ 'armv4l', 'armv5l', 'armv5el' ], # not existing arch, just for compatibility + 'armv6el': [ 'armv4l', 'armv5l', 'armv6l', 'armv5el', 'armv6el' ], # not existing arch, just for compatibility + 'armv6hl': [ 'armv4l', 'armv5l', 'armv6l', 'armv5el', 'armv6el' ], + 'armv7el': [ 'armv4l', 'armv5l', 'armv6l', 'armv7l', 'armv5el', 'armv6el', 'armv7el' ], # not existing arch, just for compatibility + 'armv7hl': [ 'armv7hl' ], # not existing arch, just for compatibility + 'armv8el': [ 'armv4l', 'armv5el', 'armv6el', 'armv7el', 'armv8el' ], # not existing arch, just for compatibility + 'armv8l': [ 'armv4l', 'armv5el', 'armv6el', 'armv7el', 'armv8el' ], # not existing arch, just for compatibility + 'armv5tel': [ 'armv4l', 'armv5el', 'armv5tel' ], 's390x': ['s390' ], 'ppc64': [ 'ppc', 'ppc64', 'ppc64p7', 'ppc64le' ], - 'ppc64le':[ 'ppc64le' ], + 'ppc64le': [ 'ppc64le' ], 'i586': [ 'i386' ], 'i686': [ 'i586', 'i386' ], 'x86_64': ['i686', 'i586', 'i386' ], @@ -116,6 +116,8 @@ def __init__(self, filename, apiurl, buildtype = 'spec', localpkgs = []): self.pacsuffix = 'deb' if self.buildtype == 'arch': self.pacsuffix = 'arch' + if self.buildtype == 'livebuild': + self.pacsuffix = 'deb' self.buildarch = root.find('arch').text if root.find('hostarch') != None: @@ -178,7 +180,7 @@ def __init__(self, node, buildarch, pacsuffix, apiurl, localpkgs = []): self.mp = {} for i in ['binary', 'package', - 'epoch', 'version', 'release', + 'epoch', 'version', 'release', 'hdrmd5', 'project', 'repository', 'preinstall', 'vminstall', 'noinstall', 'installonly', 'runscripts', 'sb2install', @@ -256,32 +258,38 @@ def __repr__(self): -def get_built_files(pacdir, pactype): - if pactype == 'rpm': +def get_built_files(pacdir, buildtype): + if buildtype == 'spec': b_built = subprocess.Popen(['find', os.path.join(pacdir, 'RPMS'), '-name', '*.rpm'], stdout=subprocess.PIPE).stdout.read().strip() s_built = subprocess.Popen(['find', os.path.join(pacdir, 'SRPMS'), '-name', '*.rpm'], stdout=subprocess.PIPE).stdout.read().strip() - elif pactype == 'kiwi': + elif buildtype == 'kiwi': b_built = subprocess.Popen(['find', os.path.join(pacdir, 'KIWI'), '-type', 'f'], stdout=subprocess.PIPE).stdout.read().strip() - elif pactype == 'deb': + s_built = '' + elif buildtype == 'dsc': b_built = subprocess.Popen(['find', os.path.join(pacdir, 'DEBS'), '-name', '*.deb'], stdout=subprocess.PIPE).stdout.read().strip() s_built = subprocess.Popen(['find', os.path.join(pacdir, 'SOURCES.DEB'), '-type', 'f'], stdout=subprocess.PIPE).stdout.read().strip() - elif pactype == 'arch': + elif buildtype == 'arch': b_built = subprocess.Popen(['find', os.path.join(pacdir, 'ARCHPKGS'), '-name', '*.pkg.tar*'], stdout=subprocess.PIPE).stdout.read().strip() s_built = '' + elif buildtype == 'livebuild': + b_built = subprocess.Popen(['find', os.path.join(pacdir, 'OTHER'), + '-name', '*.iso*'], + stdout=subprocess.PIPE).stdout.read().strip() + s_built = '' else: - print('WARNING: Unknown package type \'%s\'.' % pactype, file=sys.stderr) + print('WARNING: Unknown package type \'%s\'.' % buildtype, file=sys.stderr) b_built = '' s_built = '' return s_built, b_built @@ -311,9 +319,9 @@ def get_repo(path): return repositoryDirectory -def get_prefer_pkgs(dirs, wanted_arch, type): +def get_prefer_pkgs(dirs, wanted_arch, type, cpio): import glob - from .util import repodata, packagequery, cpio + from .util import repodata, packagequery paths = [] repositories = [] @@ -340,7 +348,9 @@ def get_prefer_pkgs(dirs, wanted_arch, type): packageQueries.add(packageQuery) for path in paths: - if path.endswith('src.rpm'): + if path.endswith('.src.rpm') or path.endswith('.nosrc.rpm'): + continue + if path.endswith('.patch.rpm') or path.endswith('.delta.rpm'): continue if path.find('-debuginfo-') > 0: continue @@ -351,21 +361,27 @@ def get_prefer_pkgs(dirs, wanted_arch, type): for name, packageQuery in packageQueries.items()) depfile = create_deps(packageQueries.values()) - cpio = cpio.CpioWrite() cpio.add('deps', '\n'.join(depfile)) - return prefer_pkgs, cpio + return prefer_pkgs def create_deps(pkgqs): """ - creates a list of requires/provides which corresponds to build's internal + creates a list of dependencies which corresponds to build's internal dependency file format """ depfile = [] for p in pkgqs: id = '%s.%s-0/0/0: ' % (p.name(), p.arch()) - depfile.append('R:%s%s' % (id, ' '.join(p.requires()))) depfile.append('P:%s%s' % (id, ' '.join(p.provides()))) + depfile.append('R:%s%s' % (id, ' '.join(p.requires()))) + d = p.conflicts() + if d: + depfile.append('C:%s%s' % (id, ' '.join(d))) + d = p.obsoletes() + if d: + depfile.append('O:%s%s' % (id, ' '.join(d))) + depfile.append('I:%s%s-%s 0-%s' % (id, p.name(), p.evr(), p.arch())) return depfile @@ -381,7 +397,7 @@ def check_trusted_projects(apiurl, projects): if not prj in trusted: print("\nThe build root needs packages from project '%s'." % prj) print("Note that malicious packages can compromise the build result or even your system.") - r = raw_input(trustprompt % { 'project':prj }) + r = raw_input(trustprompt % { 'project': prj }) if r == '1': print("adding '%s' to ~/.oscrc: ['%s']['trusted_prj']" % (prj, apiurl)) trusted.append(prj) @@ -408,9 +424,9 @@ def main(apiurl, opts, argv): build_type = os.path.splitext(build_descr)[1][1:] if os.path.basename(build_descr) == 'PKGBUILD': build_type = 'arch' - if build_type not in ['spec', 'dsc', 'kiwi', 'arch']: + if build_type not in ['spec', 'dsc', 'kiwi', 'arch', 'livebuild']: raise oscerr.WrongArgs( - 'Unknown build type: \'%s\'. Build description should end in .spec, .dsc or .kiwi.' \ + 'Unknown build type: \'%s\'. Build description should end in .spec, .dsc, .kiwi or .livebuild.' \ % build_type) if not os.path.isfile(build_descr): raise oscerr.WrongArgs('Error: build description file named \'%s\' does not exist.' % build_descr) @@ -495,6 +511,7 @@ def main(apiurl, opts, argv): if opts.shell: buildargs.append("--shell") + orig_build_root = config['build-root'] # make it possible to override configuration of the rc file for var in ['OSC_PACKAGECACHEDIR', 'OSC_SU_WRAPPER', 'OSC_BUILD_ROOT']: val = os.getenv(var) @@ -516,11 +533,15 @@ def main(apiurl, opts, argv): pacname = os.path.splitext(build_descr)[0] apihost = urlsplit(apiurl)[1] if not build_root: + build_root = config['build-root'] + if build_root == orig_build_root: + # ENV var was not set + build_root = config['api_host_options'][apiurl].get('build-root', build_root) try: - build_root = config['build-root'] % {'repo': repo, 'arch': arch, + build_root = build_root % {'repo': repo, 'arch': arch, 'project': prj, 'package': pacname, 'apihost': apihost} except: - build_root = config['build-root'] + pass cache_dir = config['packagecachedir'] % {'apihost': apihost} @@ -553,11 +574,31 @@ def main(apiurl, opts, argv): s += "%%define %s\n" % i build_descr_data = s + build_descr_data + cpiodata = None + buildenvfile = os.path.join(os.path.dirname(build_descr), "_buildenv." + repo + "." + arch) + if not os.path.isfile(buildenvfile): + buildenvfile = os.path.join(os.path.dirname(build_descr), "_buildenv") + if not os.path.isfile(buildenvfile): + buildenvfile = None + if buildenvfile: + print('Using buildenv file: %s' % os.path.basename(buildenvfile)) + from .util import cpio + if not cpiodata: + cpiodata = cpio.CpioWrite() + if opts.prefer_pkgs: print('Scanning the following dirs for local packages: %s' % ', '.join(opts.prefer_pkgs)) - prefer_pkgs, cpio = get_prefer_pkgs(opts.prefer_pkgs, arch, build_type) - cpio.add(os.path.basename(build_descr), build_descr_data) - build_descr_data = cpio.get() + from .util import cpio + if not cpiodata: + cpiodata = cpio.CpioWrite() + prefer_pkgs = get_prefer_pkgs(opts.prefer_pkgs, arch, build_type, cpiodata) + + if cpiodata: + cpiodata.add(os.path.basename(build_descr), build_descr_data) + # buildenv must come last for compatibility reasons... + if buildenvfile: + cpiodata.add("buildenv", open(buildenvfile).read()) + build_descr_data = cpiodata.get() # special handling for overlay and rsync-src/dest specialcmdopts = [] @@ -636,8 +677,8 @@ def main(apiurl, opts, argv): pkg_meta_e = None try: # take care, not to run into double trouble. - pkg_meta_e = meta_exists(metatype='pkg', path_args=(quote_plus(prj), - quote_plus(pac)), template_args=None, create_new=False, + pkg_meta_e = meta_exists(metatype='pkg', path_args=(quote_plus(prj), + quote_plus(pac)), template_args=None, create_new=False, apiurl=apiurl) except: pass @@ -760,8 +801,9 @@ class mytmpdir: """ temporary directory that removes itself""" def __init__(self, *args, **kwargs): self.name = mkdtemp(*args, **kwargs) + _rmtree = staticmethod(shutil.rmtree) def cleanup(self): - shutil.rmtree(self.name) + self._rmtree(self.name) def __del__(self): self.cleanup() def __exit__(self): @@ -875,7 +917,7 @@ def __str__(self): if not m: # short path without obs instance name m = re.match(r"obs://([^/]+)/(.+)", xml.find('source').get('path')) - project=m.group(1).replace(":",":/") + project=m.group(1).replace(":", ":/") repo=m.group(2) buildargs.append('--kiwi-parameter') buildargs.append('--add-repo') @@ -905,6 +947,17 @@ def __str__(self): else: print('WARNING: unknown packages get not verified, they can compromise your system !') + for i in bi.deps: + if i.hdrmd5: + from .util import packagequery + hdrmd5 = packagequery.PackageQuery.queryhdrmd5(i.fullfilename) + if not hdrmd5: + print("Error: cannot get hdrmd5 for %s" % i.fullfilename) + sys.exit(1) + if hdrmd5 != i.hdrmd5: + print("Error: hdrmd5 mismatch for %s: %s != %s" % (i.fullfilename, hdrmd5, i.hdrmd5)) + sys.exit(1) + print('Writing build configuration') if build_type == 'kiwi': @@ -955,6 +1008,11 @@ def __str__(self): if os.access(build_root, os.W_OK) and os.access('/dev/kvm', os.W_OK): # so let's hope there's also an fstab entry need_root = False + if config['build-kernel']: + vm_options += [ '--vm-kernel=' + config['build-kernel'] ] + if config['build-initrd']: + vm_options += [ '--vm-initrd=' + config['build-initrd'] ] + build_root += '/.mount' if config['build-memory']: @@ -1010,7 +1068,7 @@ def __str__(self): pacdir = os.path.join(build_root, pacdir) if os.path.exists(pacdir): - (s_built, b_built) = get_built_files(pacdir, bi.pacsuffix) + (s_built, b_built) = get_built_files(pacdir, bi.buildtype) print() if s_built: print(s_built) diff --git a/osc/checker.py b/osc/checker.py index 87d621b..f80a0da 100644 --- a/osc/checker.py +++ b/osc/checker.py @@ -90,7 +90,7 @@ def readkey(self, file): def check(self, pkg): # avoid errors on non rpm - if pkg[-4:] != '.rpm': + if pkg[-4:] != '.rpm': return fd = None try: diff --git a/osc/cmdln.py b/osc/cmdln.py index 8fa2cc0..da49ce6 100644 --- a/osc/cmdln.py +++ b/osc/cmdln.py @@ -393,47 +393,63 @@ def cmdloop(self, intro=None): """ self.cmdlooping = True self.preloop() - if intro is None: - intro = self.intro - if intro: - intro_str = self._str(intro) - self.stdout.write(intro_str+'\n') - self.stop = False - retval = None - while not self.stop: - if self.cmdqueue: - argv = self.cmdqueue.pop(0) - assert isinstance(argv, (list, tuple)), \ - "item on 'cmdqueue' is not a sequence: %r" % argv - else: - if self.use_rawinput: - try: - try: - #python 2.x - line = raw_input(self._prompt_str) - except NameError: - line = input(self._prompt_str) - except EOFError: - line = 'EOF' + if self.use_rawinput and self.completekey: + try: + import readline + self.old_completer = readline.get_completer() + readline.set_completer(self.complete) + readline.parse_and_bind(self.completekey+": complete") + except ImportError: + pass + try: + if intro is None: + intro = self.intro + if intro: + intro_str = self._str(intro) + self.stdout.write(intro_str+'\n') + self.stop = False + retval = None + while not self.stop: + if self.cmdqueue: + argv = self.cmdqueue.pop(0) + assert isinstance(argv, (list, tuple)), \ + "item on 'cmdqueue' is not a sequence: %r" % argv else: - self.stdout.write(self._prompt_str) - self.stdout.flush() - line = self.stdin.readline() - if not len(line): - line = 'EOF' + if self.use_rawinput: + try: + try: + #python 2.x + line = raw_input(self._prompt_str) + except NameError: + line = input(self._prompt_str) + except EOFError: + line = 'EOF' else: - line = line[:-1] # chop '\n' - argv = line2argv(line) - try: - argv = self.precmd(argv) - retval = self.onecmd(argv) - self.postcmd(argv) - except: - if not self.cmdexc(argv): - raise - retval = 1 - self.lastretval = retval - self.postloop() + self.stdout.write(self._prompt_str) + self.stdout.flush() + line = self.stdin.readline() + if not len(line): + line = 'EOF' + else: + line = line[:-1] # chop '\n' + argv = line2argv(line) + try: + argv = self.precmd(argv) + retval = self.onecmd(argv) + self.postcmd(argv) + except: + if not self.cmdexc(argv): + raise + retval = 1 + self.lastretval = retval + self.postloop() + finally: + if self.use_rawinput and self.completekey: + try: + import readline + readline.set_completer(self.old_completer) + except ImportError: + pass self.cmdlooping = False return retval @@ -516,7 +532,7 @@ def parseline(self, line): elif line[0] == '?': line = 'help ' + line[1:] i, n = 0, len(line) - while i < n and line[i] in self.identchars: + while i < n and line[i] in self.identchars: i = i+1 cmd, arg = line[:i], line[i:].strip() return cmd, arg, line @@ -574,7 +590,7 @@ def do_help(self, argv): doc = self.__class__.__doc__ # try class docstring if doc is None: # Try to provide some reasonable useful default help. - if self.cmdlooping: + if self.cmdlooping: prefix = "" else: prefix = self.name+' ' @@ -739,7 +755,7 @@ def _help_get_command_list(self): token2canonical = self._get_canonical_map() aliases = {} for token, cmdname in token2canonical.items(): - if token == cmdname: + if token == cmdname: continue aliases.setdefault(cmdname, []).append(token) @@ -803,7 +819,7 @@ def _help_preprocess_help_list(self, help, cmdname=None): helpnames = {} token2cmdname = self._get_canonical_map() for attr in self.get_names(): - if not attr.startswith("help_"): + if not attr.startswith("help_"): continue helpname = attr[5:] if helpname not in token2cmdname: @@ -854,9 +870,9 @@ def _help_preprocess_cmd_usage(self, help, cmdname=None): # Adjust argcount for possible *args and **kwargs arguments. argcount = co_argcount - if co_flags & CO_FLAGS_ARGS: + if co_flags & CO_FLAGS_ARGS: argcount += 1 - if co_flags & CO_FLAGS_KWARGS: + if co_flags & CO_FLAGS_KWARGS: argcount += 1 # Determine the usage string. @@ -937,9 +953,9 @@ def _get_canonical_map(self): token2canonical = {} cmd2funcname = {} # use a dict to strip duplicates for attr in self.get_names(): - if attr.startswith("do_"): + if attr.startswith("do_"): cmdname = attr[3:] - elif attr.startswith("_do_"): + elif attr.startswith("_do_"): cmdname = attr[4:] else: continue @@ -1263,7 +1279,7 @@ def _format_linedata(linedata, indent, indent_width): SPACING = 3 MAX_NAME_WIDTH = 15 - NAME_WIDTH = min(max([len(s) for s,d in linedata]), MAX_NAME_WIDTH) + NAME_WIDTH = min(max([len(s) for s, d in linedata]), MAX_NAME_WIDTH) DOC_WIDTH = WIDTH - NAME_WIDTH - SPACING for namestr, doc in linedata: line = indent + namestr @@ -1371,12 +1387,12 @@ def line2argv(line): i = -1 while True: i += 1 - if i >= len(line): + if i >= len(line): break ch = line[i] if ch == "\\": # escaped char always added to arg, regardless of state - if arg is None: + if arg is None: arg = "" i += 1 arg += line[i] @@ -1394,11 +1410,11 @@ def line2argv(line): arg += ch elif state == "default": if ch == '"': - if arg is None: + if arg is None: arg = "" state = "double-quoted" elif ch == "'": - if arg is None: + if arg is None: arg = "" state = "single-quoted" elif ch in string.whitespace: @@ -1406,7 +1422,7 @@ def line2argv(line): argv.append(arg) arg = None else: - if arg is None: + if arg is None: arg = "" arg += ch if arg is not None: @@ -1485,7 +1501,7 @@ def _dedentlines(lines, tabsize=8, skip_first_line=False): break else: continue # skip all-whitespace lines - if DEBUG: + if DEBUG: print("dedent: indent=%d: %r" % (indent, line)) if margin is None: margin = indent @@ -1496,7 +1512,7 @@ def _dedentlines(lines, tabsize=8, skip_first_line=False): if margin is not None and margin > 0: for i, line in enumerate(lines): - if i == 0 and skip_first_line: + if i == 0 and skip_first_line: continue removed = 0 for j, ch in enumerate(line): @@ -1505,7 +1521,7 @@ def _dedentlines(lines, tabsize=8, skip_first_line=False): elif ch == '\t': removed += tabsize - (removed % tabsize) elif ch in '\r\n': - if DEBUG: + if DEBUG: print("dedent: %r: EOL -> strip up to EOL" % line) lines[i] = lines[i][j:] break diff --git a/osc/commandline.py b/osc/commandline.py index d48cecd..585ef07 100644 --- a/osc/commandline.py +++ b/osc/commandline.py @@ -47,7 +47,7 @@ * http://en.opensuse.org/openSUSE:Build_Service_Tutorial * http://en.opensuse.org/openSUSE:OSC .PP -You can modify osc commands, or roll you own, via the plugin API: +You can modify osc commands, or roll your own, via the plugin API: * http://en.opensuse.org/openSUSE:OSC_plugins .SH AUTHOR osc was written by several authors. This man page is automatically generated. @@ -67,7 +67,7 @@ class Osc(cmdln.Cmdln): * http://en.opensuse.org/openSUSE:Build_Service_Tutorial * http://en.opensuse.org/openSUSE:OSC - You can modify osc commands, or roll you own, via the plugin API: + You can modify osc commands, or roll your own, via the plugin API: * http://en.opensuse.org/openSUSE:OSC_plugins """ name = 'osc' @@ -150,7 +150,7 @@ def postoptparse(self, try_again = True): conf.write_initial_config(e.file, config) print('done', file=sys.stderr) - if try_again: + if try_again: self.postoptparse(try_again = False) except oscerr.ConfigMissingApiurl as e: print(e.msg, file=sys.stderr) @@ -158,7 +158,7 @@ def postoptparse(self, try_again = True): user = raw_input('Username: ') passwd = getpass.getpass() conf.add_section(e.file, e.url, user, passwd) - if try_again: + if try_again: self.postoptparse(try_again = False) self.options.verbose = conf.config['verbose'] @@ -182,7 +182,7 @@ def get_api_url(self): ## check for Stale NFS file handle: '.' try: os.stat('.') - except Exception as ee: + except Exception as ee: e = ee print("os.getcwd() failed: ", e, file=sys.stderr) sys.exit(1) @@ -304,7 +304,7 @@ def do_list(self, subcmd, opts, *args): pass if len(args) > 0: project = args[0] - if project == '/': + if project == '/': project = None if project == '.': cwd = os.getcwd() @@ -340,7 +340,7 @@ def do_list(self, subcmd, opts, *args): if opts.binaries and opts.expand: raise oscerr.WrongOptions('Sorry, --binaries and --expand are mutual exclusive.') - apiurl = self.get_api_url() + apiurl = self.get_api_url() # list binaries if opts.binaries: @@ -397,9 +397,7 @@ def do_list(self, subcmd, opts, *args): if opts.verbose: if self.options.verbose: print('Sorry, the --verbose option is not implemented for projects.', file=sys.stderr) - if opts.expand: - raise oscerr.WrongOptions('Sorry, the --expand option is not implemented for projects.') - for pkg in meta_get_packagelist(apiurl, project, opts.deleted): + for pkg in meta_get_packagelist(apiurl, project, deleted = opts.deleted, expand = opts.expand): print(pkg) elif len(args) == 2 or len(args) == 3: @@ -428,7 +426,7 @@ def do_list(self, subcmd, opts, *args): print_not_found = False else: print('\n'.join(l)) - if opts.expand or opts.unexpand or not link_seen: + if opts.expand or opts.unexpand or not link_seen: break m = show_files_meta(apiurl, project, package) li = Linkinfo() @@ -461,7 +459,7 @@ def do_patchinfo(self, subcmd, opts, *args): ${cmd_option_list} """ - apiurl = self.get_api_url() + apiurl = self.get_api_url() project_dir = localdir = os.getcwd() patchinfo = 'patchinfo' if len(args) == 0: @@ -509,7 +507,7 @@ def do_patchinfo(self, subcmd, opts, *args): f = http_POST(url) # CAUTION: - # Both conf.config['checkout_no_colon'] and conf.config['checkout_rooted'] + # Both conf.config['checkout_no_colon'] and conf.config['checkout_rooted'] # fool this test: if is_package_dir(localdir): pac = Package(localdir) @@ -695,13 +693,13 @@ def do_meta(self, subcmd, opts, *args): osc meta prj PRJ osc meta pkg PRJ PKG osc meta pkg PRJ PKG -e - osc meta attribute PRJ [PKG [SUBPACKAGE]] [--attribute ATTRIBUTE] [--create|--delete|--set [value_list]] Usage: - osc meta ARGS... - osc meta -e|--edit ARGS... - osc meta -F|--file ARGS... + osc meta ARGS... + osc meta -e|--edit ARGS... + osc meta -F|--file ARGS... osc meta pattern --delete PRJ PATTERN + osc meta attribute PRJ [PKG [SUBPACKAGE]] [--attribute ATTRIBUTE] [--create|--delete|--set [value_list]] ${cmd_option_list} """ @@ -794,7 +792,7 @@ def do_meta(self, subcmd, opts, *args): elif cmd == 'pkg': sys.stdout.write(''.join(show_package_meta(apiurl, project, package))) elif cmd == 'attribute': - sys.stdout.write(''.join(show_attribute_meta(apiurl, project, package, subpackage, + sys.stdout.write(''.join(show_attribute_meta(apiurl, project, package, subpackage, opts.attribute, opts.attribute_defaults, opts.attribute_project))) elif cmd == 'prjconf': sys.stdout.write(''.join(show_project_conf(apiurl, project))) @@ -950,6 +948,8 @@ def do_meta(self, subcmd, opts, *args): help='never remove source package on accept, but update its content') @cmdln.option('--no-update', action='store_true', help='never touch source package on accept (will break source links)') + @cmdln.option('--update-link', action='store_true', + help='This transfers the source including the _link file.') @cmdln.option('-d', '--diff', action='store_true', help='show diff only instead of creating the actual request') @cmdln.option('--yes', action='store_true', @@ -1028,9 +1028,12 @@ def do_submitrequest(self, subcmd, opts, *args): sr_ids = [] # for single request actionxml = "" - options_block = "" + options_block = "" if src_update: - options_block = """%s """ % (src_update) + options_block += """%s""" % (src_update) + if opts.update_link: + options_block + """true """ + options_block += "" # loop via all packages for checking their state for p in meta_get_packagelist(apiurl, project): @@ -1202,7 +1205,7 @@ def do_submitrequest(self, subcmd, opts, *args): rev = root.get('rev') else: if linkinfo.get('project') != dst_project or linkinfo.get('package') != dst_package: - # the submit target is not link target. use merged md5sum references to + # the submit target is not link target. use merged md5sum references to # avoid not mergable sources when multiple request from same source get created. rev = root.get('srcmd5') @@ -1244,7 +1247,8 @@ def do_submitrequest(self, subcmd, opts, *args): result = create_submit_request(apiurl, src_project, src_package, dst_project, dst_package, - opts.message, orev=rev, src_update=src_update) + opts.message, orev=rev, + src_update=src_update, dst_updatelink=opts.update_link) if supersede_existing: for req in reqs: change_request_state(apiurl, req.reqid, 'superseded', @@ -1288,7 +1292,6 @@ def _submit_request(self, args, opts, options_block): pi = [] pac = [] targetprojects = [] - rdiffmsg = [] # loop via all packages for checking their state for p in meta_get_packagelist(apiurl, project): if p.startswith("_patchinfo:"): @@ -1316,21 +1319,10 @@ def _submit_request(self, args, opts, options_block): if rdiff != '': targetprojects.append(t) pac.append(p) - rdiffmsg.append("old: %s/%s\nnew: %s/%s\n%s" % (t, p, project, p, rdiff)) else: print("Skipping package ", p, " since it has no difference with the target package.") else: print("Skipping package ", p, " since it is a source link pointing inside the project.") - if opts.diff: - print(''.join(rdiffmsg)) - sys.exit(0) - - if not opts.yes: - if pi: - print("Submitting patchinfo ", ', '.join(pi), " to ", ', '.join(targetprojects)) - print("\nEverything fine? Can we create the requests ? [y/n]") - if sys.stdin.read(1) != "y": - sys.exit("Aborted...") # loop via all packages to do the action for p in pac: @@ -1345,7 +1337,7 @@ def _submit_request(self, args, opts, options_block): (project, p, t, p, options_block) actionxml += s - return actionxml + return actionxml, [] elif len(args) <= 2: # try using the working copy at hand @@ -1399,45 +1391,29 @@ def _submit_request(self, args, opts, options_block): of the package %s primarily takes place. Please submit there instead, or use --nodevelproject to force direct submission.""" \ % (devloc, dst_package)) - if not opts.diff: - sys.exit(1) - - rdiff = None - if opts.diff: - try: - rdiff = 'old: %s/%s\nnew: %s/%s\n' % (dst_project, dst_package, src_project, src_package) - rdiff += server_diff(apiurl, - dst_project, dst_package, opts.revision, - src_project, src_package, None, True) - except: - rdiff = '' - if opts.diff: - run_pager(rdiff) - else: - reqs = get_request_list(apiurl, dst_project, dst_package, req_type='submit', req_state=['new','review']) - user = conf.get_apiurl_usr(apiurl) - myreqs = [ i for i in reqs if i.state.who == user ] - repl = 'y' - if len(myreqs) > 0 and not opts.yes: - print('You already created the following submit request: %s.' % \ - ', '.join([i.reqid for i in myreqs ])) - repl = raw_input('Supersede the old requests? (y/n/c) ') - if repl.lower() == 'c': - print('Aborting', file=sys.stderr) - sys.exit(1) + sys.exit(1) - actionxml = """ %s """ % \ - (src_project, src_package, opts.revision or show_upstream_rev(apiurl, src_project, src_package), dst_project, dst_package, options_block) - if repl.lower() == 'y': - for req in myreqs: - change_request_state(apiurl, req.reqid, 'superseded', - 'superseded by %s' % result, result) + reqs = get_request_list(apiurl, dst_project, dst_package, req_type='submit', req_state=['new', 'review']) + user = conf.get_apiurl_usr(apiurl) + myreqs = [ i for i in reqs if i.state.who == user and i.reqid != opts.supersede ] + repl = 'y' + if len(myreqs) > 0 and not opts.yes: + print('You already created the following submit request: %s.' % \ + ', '.join([i.reqid for i in myreqs ])) + repl = raw_input('Supersede the old requests? (y/n/c) ') + if repl.lower() == 'c': + print('Aborting', file=sys.stderr) + sys.exit(1) + elif repl.lower() != 'y': + myreqs = [] - if opts.supersede: - change_request_state(apiurl, opts.supersede, 'superseded', '', result) + actionxml = """ %s """ % \ + (src_project, src_package, opts.revision or show_upstream_rev(apiurl, src_project, src_package), dst_project, dst_package, options_block) + if opts.supersede: + myreqs.append(opts.supersede) - #print 'created request id', result - return actionxml + #print 'created request id', result + return actionxml, myreqs def _delete_request(self, args, opts): if len(args) < 1: @@ -1563,7 +1539,7 @@ def _set_bugowner(self, args, opts): package = """package="%s" """ % (args[2]) if user.startswith('group:'): - group = user.replace('group:','') + group = user.replace('group:', '') actionxml = """ """ % \ (project, package, group) if get_group(apiurl, group) == None: @@ -1577,7 +1553,7 @@ def _set_bugowner(self, args, opts): return actionxml - @cmdln.option('-a', '--action', action='callback', callback = _actionparser,dest = 'actions', + @cmdln.option('-a', '--action', action='callback', callback = _actionparser, dest = 'actions', help='specify action type of a request, can be : submit/delete/change_devel/add_role/set_bugowner') @cmdln.option('-m', '--message', metavar='TEXT', help='specify message TEXT') @@ -1594,8 +1570,6 @@ def _set_bugowner(self, args, opts): help='never remove source package on accept, but update its content') @cmdln.option('--no-update', action='store_true', help='never touch source package on accept (will break source links)') - @cmdln.option('-d', '--diff', action='store_true', - help='show diff only instead of creating the actual request') @cmdln.option('--yes', action='store_true', help='proceed without asking.') @cmdln.alias("creq") @@ -1603,10 +1577,10 @@ def do_createrequest(self, subcmd, opts, *args): """${cmd_name}: create multiple requests with a single command usage: - osc creq [OPTIONS] [ - -a submit SOURCEPRJ SOURCEPKG DESTPRJ [DESTPKG] - -a delete PROJECT [PACKAGE] - -a change_devel PROJECT PACKAGE DEVEL_PROJECT [DEVEL_PACKAGE] + osc creq [OPTIONS] [ + -a submit SOURCEPRJ SOURCEPKG DESTPRJ [DESTPKG] + -a delete PROJECT [PACKAGE] + -a change_devel PROJECT PACKAGE DEVEL_PROJECT [DEVEL_PACKAGE] -a add_me ROLE PROJECT [PACKAGE] -a add_group GROUP ROLE PROJECT [PACKAGE] -a add_role USER ROLE PROJECT [PACKAGE] @@ -1639,11 +1613,14 @@ def do_createrequest(self, subcmd, opts, *args): i = 0 actionsxml = "" + supersede = [] for ai in opts.actions: if ai == 'submit': args = opts.actiondata[i] i = i+1 - actionsxml += self._submit_request(args, opts, options_block) + actions, to_supersede = self._submit_request(args, opts, options_block) + actionsxml += actions + supersede.extend(to_supersede) elif ai == 'delete': args = opts.actiondata[i] actionsxml += self._delete_request(args, opts) @@ -1683,7 +1660,11 @@ def do_createrequest(self, subcmd, opts, *args): f = http_POST(u, data=xml) root = ET.parse(f).getroot() - return root.get('id') + rid = root.get('id') + for srid in supersede: + change_request_state(apiurl, srid, 'superseded', + 'superseded by %s' % rid, rid) + return rid @cmdln.option('-m', '--message', metavar='TEXT', @@ -1699,13 +1680,14 @@ def do_requestmaintainership(self, subcmd, opts, *args): """${cmd_name}: requests to add user as maintainer or bugowner usage: - osc requestmaintainership # for current user in checked out package - osc requestmaintainership USER # for specified user in checked out package - osc requestmaintainership PROJECT # for current user if cwd is not a checked out package - osc requestmaintainership PROJECT PACKAGE # for current user - osc requestmaintainership PROJECT PACKAGE USER # request for specified user + osc requestmaintainership # for current user in checked out package + osc requestmaintainership USER # for specified user in checked out package + osc requestmaintainership PROJECT # for current user if cwd is not a checked out package + osc requestmaintainership PROJECT PACKAGE # for current user + osc requestmaintainership PROJECT PACKAGE USER # request for specified user + osc requestmaintainership PROJECT PACKAGE group:NAME # request for specified group - osc requestbugownership ... # accepts same parameters but uses bugowner role + osc requestbugownership ... # accepts same parameters but uses bugowner role ${cmd_option_list} """ @@ -1746,7 +1728,15 @@ def do_requestmaintainership(self, subcmd, opts, *args): opts.message = edit_message() r = Request() - if role == 'bugowner': + if user.startswith('group:'): + group = user.replace('group:', '') + if role == 'bugowner': + r.add_action('set_bugowner', tgt_project=project, tgt_package=package, + group_name=group) + else: + r.add_action('add_role', tgt_project=project, tgt_package=package, + group_name=group, group_role=role) + elif role == 'bugowner': r.add_action('set_bugowner', tgt_project=project, tgt_package=package, person_name=user) else: @@ -1795,7 +1785,7 @@ def do_deleterequest(self, subcmd, opts, *args): elif is_package_dir(os.getcwd()): project = store_read_project(os.curdir) package = store_read_package(os.curdir) - else: + else: raise oscerr.WrongArgs('Please specify at least a project.') if opts.repository: @@ -1806,7 +1796,7 @@ def do_deleterequest(self, subcmd, opts, *args): if package is not None: footer = textwrap.TextWrapper(width = 66).fill( 'please explain why you like to delete package %s of project %s' - % (package,project)) + % (package, project)) else: footer = textwrap.TextWrapper(width = 66).fill( 'please explain why you like to delete project %s' % project) @@ -1828,7 +1818,7 @@ def do_deleterequest(self, subcmd, opts, *args): def do_changedevelrequest(self, subcmd, opts, *args): """${cmd_name}: Create request to change the devel package definition. - [See http://en.opensuse.org/openSUSE:Build_Service_Collaboration + [See http://en.opensuse.org/openSUSE:Build_Service_Collaboration for information on this topic.] See the "request" command for showing and modifing existing requests. @@ -1858,7 +1848,7 @@ def do_changedevelrequest(self, subcmd, opts, *args): import textwrap footer = textwrap.TextWrapper(width = 66).fill( 'please explain why you like to change the devel project of %s/%s to %s/%s' - % (project,package,devel_project,devel_package)) + % (project, package, devel_project, devel_package)) opts.message = edit_message(footer) r = Request() @@ -1951,10 +1941,13 @@ def do_request(self, subcmd, opts, *args): "checkout" will checkout the request's source package ("submit" requests only). + "priorize" change the prioritity of a request to either "critical", "important", "moderate" or "low" + + The 'review' command has the following sub commands: "list" lists open requests that need to be reviewed by the - specified user or group + specified user or group "add" adds a person or group as reviewer to a request @@ -1975,6 +1968,7 @@ def do_request(self, subcmd, opts, *args): osc request setincident [-m TEXT] ID INCIDENT osc request supersede [-m TEXT] ID SUPERSEDING_ID osc request approvenew [-m TEXT] PROJECT + osc request priorize [-m TEXT] ID PRIORITY osc request checkout/co ID osc request clone [-m TEXT] ID @@ -2008,29 +2002,31 @@ def do_request(self, subcmd, opts, *args): if opts.state == '': opts.state = 'all' - if opts.state == '': + if opts.state == '' and subcmd != 'review': opts.state = 'declined,new,review' if args[0] == 'help': return self.do_help(['help', 'request']) - cmds = ['list', 'log', 'show', 'decline', 'reopen', 'clone', 'accept', 'approvenew', 'wipe', 'setincident', 'supersede', 'revoke', 'checkout', 'co'] + cmds = ['list', 'ls', 'log', 'show', 'decline', 'reopen', 'clone', 'accept', 'approvenew', 'wipe', 'setincident', 'supersede', 'revoke', 'checkout', 'co', 'priorize'] if subcmd != 'review' and args[0] not in cmds: raise oscerr.WrongArgs('Unknown request action %s. Choose one of %s.' \ - % (args[0],', '.join(cmds))) + % (args[0], ', '.join(cmds))) cmds = ['show', 'list', 'add', 'decline', 'accept', 'reopen', 'supersede'] if subcmd == 'review' and args[0] not in cmds: raise oscerr.WrongArgs('Unknown review action %s. Choose one of %s.' \ - % (args[0],', '.join(cmds))) + % (args[0], ', '.join(cmds))) cmd = args[0] del args[0] + if cmd == 'ls': + cmd = "list" apiurl = self.get_api_url() if cmd in ['list']: min_args, max_args = 0, 2 - elif cmd in ['supersede', 'setincident']: + elif cmd in ['supersede', 'setincident', 'priorize']: min_args, max_args = 2, 2 else: min_args, max_args = 1, 1 @@ -2067,6 +2063,9 @@ def do_request(self, subcmd, opts, *args): elif cmd == 'setincident': reqid = args[0] incident = args[1] + elif cmd == 'priorize': + reqid = args[0] + priority = args[1] elif cmd in ['log', 'add', 'show', 'decline', 'reopen', 'clone', 'accept', 'wipe', 'revoke', 'checkout', 'co']: reqid = args[0] @@ -2082,6 +2081,13 @@ def do_request(self, subcmd, opts, *args): r = http_POST(url, data=opts.message) print(ET.parse(r).getroot().get('code')) + # change priority + elif cmd == 'priorize': + query = { 'cmd': 'setpriority', 'priority': priority } + url = makeurl(apiurl, ['request', reqid], query) + r = http_POST(url, data=opts.message) + print(ET.parse(r).getroot().get('code')) + # add new reviewer to existing request elif cmd in ['add'] and subcmd == 'review': query = { 'cmd': 'addreview' } @@ -2108,13 +2114,10 @@ def do_request(self, subcmd, opts, *args): results = get_request_list(apiurl, project, package, '', ['new']) else: state_list = opts.state.split(',') + if state_list == ['']: + state_list = () if opts.all: state_list = ['all'] - if subcmd == 'review': - # is there a special reason why we do not respect the passed states? - state_list = ['new'] - elif opts.state == 'all': - state_list = ['all'] else: for s in state_list: if not s in states and not s == 'all': @@ -2153,7 +2156,7 @@ def do_request(self, subcmd, opts, *args): print('No results') return - results.sort(reverse=True) + # we must not sort the results here, since the api is doing it already "the right way" days = opts.days or conf.config['request_list_days'] since = '' try: @@ -2310,7 +2313,7 @@ def do_request(self, subcmd, opts, *args): print('Result of change request state: %s' % r) except HTTPError as e: print(e, file=sys.stderr) - details = e.headers.get('X-Opensuse-Errorcode') + details = e.hdrs.get('X-Opensuse-Errorcode') if details: print(details, file=sys.stderr) root = ET.fromstring(e.read()) @@ -2322,6 +2325,8 @@ def do_request(self, subcmd, opts, *args): print('Revoking it ...') r = change_request_state(apiurl, reqid, 'revoked', opts.message or '', supersed=supersedid, force=opts.force) + sys.exit(1) + # check for devel instances after accepted requests if cmd in ['accept']: @@ -2753,7 +2758,7 @@ def do_copypac(self, subcmd, opts, *args): @cmdln.option('--set-release', metavar='RELEASETAG', help='rename binaries during release using this release tag') def do_release(self, subcmd, opts, *args): - """${cmd_name}: Release sources and binaries + """${cmd_name}: Release sources and binaries This command is used to transfer sources and binaries without rebuilding them. It requires defined release targets set to trigger="manual". Please refer the @@ -2972,6 +2977,8 @@ def do_createincident(self, subcmd, opts, *args): help='Use this attribute to find default maintenance project (default is OBS:MaintenanceProject)') @cmdln.option('-m', '--message', metavar='TEXT', help='specify message TEXT') + @cmdln.option('--release-project', metavar='RELEASEPROJECT', + help='Specify the release project') @cmdln.option('--no-cleanup', action='store_true', help='do not remove source project on accept') @cmdln.option('--cleanup', action='store_true', @@ -2995,6 +3002,9 @@ def do_maintenancerequest(self, subcmd, opts, *args): osc maintenancerequest [ SOURCEPROJECT [ SOURCEPACKAGES RELEASEPROJECT ] ] ${cmd_option_list} """ + #FIXME: the follow syntax would make more sense and would obsolete the --release-project parameter + # but is incompatible with the current one + # osc maintenancerequest [ SOURCEPROJECT [ RELEASEPROJECT [ SOURCEPACKAGES ] ] args = slash_split(args) apiurl = self.get_api_url() @@ -3023,6 +3033,9 @@ def do_maintenancerequest(self, subcmd, opts, *args): if source_project.startswith(default_branch): opt_sourceupdate = 'cleanup' + if opts.release_project: + release_project = opts.release_project + if opts.incident_project: target_project = opts.incident_project else: @@ -3188,7 +3201,7 @@ def do_branch(self, subcmd, opts, *args): ${cmd_option_list} """ - if subcmd == 'getpac' or subcmd == 'branchco' or subcmd == 'bco': + if subcmd == 'getpac' or subcmd == 'branchco' or subcmd == 'bco': opts.checkout = True args = slash_split(args) tproject = tpackage = None @@ -3347,7 +3360,7 @@ def do_rdelete(self, subcmd, opts, *args): ## FIXME: core.py:commitDelPackage() should have something similar rlist = get_request_list(apiurl, prj, pkg) - for rq in rlist: + for rq in rlist: print(rq) if len(rlist) >= 1 and not opts.force: print('Package has pending requests. Deleting the package will break them. '\ @@ -3431,6 +3444,7 @@ def do_deleteprj(self, subcmd, opts, project): sys.exit(1) @cmdln.alias('metafromspec') + @cmdln.alias('updatepkgmetafromspec') @cmdln.option('', '--specfile', metavar='FILE', help='Path to specfile. (if you pass more than working copy this option is ignored)') def do_updatepacmetafromspec(self, subcmd, opts, *args): @@ -3481,11 +3495,11 @@ def do_diff(self, subcmd, opts, *args): Default: all files. osc diff --link - osc linkdiff + osc linkdiff Compare current checkout directory against the link base. - osc diff --link PROJ PACK - osc linkdiff PROJ PACK + osc diff --link PROJ PACK + osc linkdiff PROJ PACK Compare a package against the link base (ignoring working copy changes). ${cmd_option_list} @@ -3786,7 +3800,7 @@ def do_pdiff(self, subcmd, opts, *args): return 1 if not noparentok and not self._pdiff_package_exists(apiurl, parent_project, parent_package): - self._pdiff_raise_non_existing_package(parent_project, parent_package, + self._pdiff_raise_non_existing_package(parent_project, parent_package, msg = 'Parent for %s/%s (%s/%s) does not exist.' % \ (project, package, parent_project, parent_package)) @@ -3951,7 +3965,7 @@ def do_prdiff(self, subcmd, opts, *args): def do_install(self, subcmd, opts, *args): """${cmd_name}: install a package after build via zypper in -r - Not implemented here. Please try + Not implemented here. Please try http://software.opensuse.org/search?q=osc-plugin-install&include_home=true @@ -4053,7 +4067,7 @@ def do_checkout(self, subcmd, opts, *args): osc co PACKAGE # check out PACKAGE from project with the result of rpm -q --qf '%%{DISTURL}\\n' PACKAGE - osc co obs://API/PROJECT/PLATFORM/REVISION-PACKAGE + osc co obs://API/PROJECT/PLATFORM/REVISION-PACKAGE ${cmd_option_list} """ @@ -4068,7 +4082,7 @@ def do_checkout(self, subcmd, opts, *args): + self.get_cmd_help('checkout')) # XXX: this too openSUSE-setup specific... - # FIXME: this should go into ~jw/patches/osc/osc.proj_pack_20101201.diff + # FIXME: this should go into ~jw/patches/osc/osc.proj_pack_20101201.diff # to be available to all subcommands via @cmdline.prep(proj_pack) # obs://build.opensuse.org/openSUSE:11.3/standard/fc6c25e795a89503e99d59da5dc94a79-screen m = re.match(r"obs://([^/]+)/(\S+)/([^/]+)/([A-Fa-f\d]+)\-(\S+)", args[0]) @@ -4192,6 +4206,8 @@ def do_status(self, subcmd, opts, *args): 'M' Modified '?' item is not under version control '!' item is missing (removed by non-osc command) or incomplete + 'S' item is skipped (item exceeds a file size limit or is _service:* file) + 'F' Frozen (use "osc pull" to merge conflicts) (package-only state) examples: osc st @@ -4225,7 +4241,9 @@ def do_status(self, subcmd, opts, *args): # state is != ' ' lines.append(statfrmt(st, os.path.normpath(os.path.join(prj.dir, pac)))) continue - if st == ' ' and opts.verbose or st != ' ': + if p.isfrozen(): + lines.append(statfrmt('F', os.path.normpath(os.path.join(prj.dir, pac)))) + elif st == ' ' and opts.verbose or st != ' ': lines.append(statfrmt(st, os.path.normpath(os.path.join(prj.dir, pac)))) states = p.get_status(opts.show_excluded, *excl_states) for st, filename in sorted(states, lambda x, y: cmp(x[1], y[1])): @@ -4330,9 +4348,10 @@ def do_addremove(self, subcmd, opts, *args): pacs = findpacs(args) for p in pacs: - p.todo = list(set(p.filenamelist + p.filenamelist_unvers + p.to_be_added)) - for filename in p.todo: - if os.path.isdir(filename): + todo = list(set(p.filenamelist + p.filenamelist_unvers + p.to_be_added)) + for filename in todo: + abs_filename = os.path.join(p.absdir, filename) + if os.path.isdir(abs_filename): continue # ignore foo.rXX, foo.mine for files which are in 'C' state if os.path.splitext(filename)[0] in p.in_conflict: @@ -4341,6 +4360,9 @@ def do_addremove(self, subcmd, opts, *args): if state == '?': # TODO: should ignore typical backup files suffix ~ or .orig p.addfile(filename) + elif state == 'D' and os.path.isfile(abs_filename): + # if the "deleted" file exists in the wc, track it again + p.addfile(filename) elif state == '!': p.delete_file(filename) print(statfrmt('D', getTransActPath(os.path.join(p.dir, filename)))) @@ -4463,7 +4485,6 @@ def do_commit(self, subcmd, opts, *args): store_unlink_file(p.absdir, '_commit_msg') else: for p in pacs: - p = Package(pac) if not p.todo: p.todo = p.filenamelist + p.filenamelist_unvers p.todo.sort() @@ -4700,10 +4721,10 @@ def do_distributions(self, subcmd, opts, *args): """${cmd_name}: Shows all available distributions This command shows the available distributions. For active distributions - it shows the name, project and name of the repository and a suggested default repository name. + it shows the name, project and name of the repository and a suggested default repository name. usage: - osc distributions + osc distributions ${cmd_option_list} """ @@ -5099,9 +5120,11 @@ def do_localbuildlog(self, subcmd, opts, *args): self.print_repos() raise oscerr.WrongArgs('Wrong number of arguments.') + # TODO: refactor/unify buildroot calculation and move it to core.py buildroot = os.environ.get('OSC_BUILD_ROOT', conf.config['build-root']) + apihost = urlsplit(self.get_api_url())[1] buildroot = buildroot % {'project': project, 'package': package, - 'repo': repo, 'arch': arch} + 'repo': repo, 'arch': arch, 'apihost': apihost} offset = 0 if opts.offset: offset = int(opts.offset) @@ -5317,8 +5340,8 @@ def do_buildinfo(self, subcmd, opts, *args): elif opts.prefer_pkgs: from .build import get_prefer_pkgs print('Scanning the following dirs for local packages: %s' % ', '.join(opts.prefer_pkgs)) - prefer_pkgs, cpio = get_prefer_pkgs(opts.prefer_pkgs, arch, os.path.splitext(args[2])[1]) - cpio.add(os.path.basename(args[2]), build_descr_data) + prefer_pkgs, cpio = get_prefer_pkgs(opts.prefer_pkgs, arch, os.path.splitext(build_descr)[1]) + cpio.add(os.path.basename(build_descr), build_descr_data) build_descr_data = cpio.get() print(''.join(get_buildinfo(apiurl, @@ -5338,7 +5361,7 @@ def do_buildconfig(self, subcmd, opts, *args): which is directly readable by the build script. It contains RPM macros and BuildRequires expansions, for example. - The argument REPOSITORY an be taken from the first column of the + The argument REPOSITORY an be taken from the first column of the 'osc repos' output. usage: @@ -5410,7 +5433,7 @@ def do_repositories(self, subcmd, opts, *args): disabled = show_package_disabled_repos(apiurl, project, package) if subcmd == 'repos_only': - for repo in get_repositories_of_project(apiurl, project): + for repo in get_repositories_of_project(apiurl, project): if (disabled is None) or ((disabled is not None) and (repo not in disabled)): print(repo) else: @@ -5436,7 +5459,7 @@ def parse_repoarchdescr(self, args, noinit = False, alternative_project = None, for subarch in osc.build.can_also_build.get(mainarch): all_archs.append(subarch) for arg in args: - if arg.endswith('.spec') or arg.endswith('.dsc') or arg.endswith('.kiwi') or arg == 'PKGBUILD': + if arg.endswith('.spec') or arg.endswith('.dsc') or arg.endswith('.kiwi') or arg.endswith('.livebuild') or arg == 'PKGBUILD': arg_descr = arg else: if (arg == osc.build.hostarch or arg in all_archs) and arg_arch is None: @@ -5447,7 +5470,10 @@ def parse_repoarchdescr(self, args, noinit = False, alternative_project = None, elif not arg_repository: arg_repository = arg else: - raise oscerr.WrongArgs('unexpected argument: \'%s\'' % arg) +# raise oscerr.WrongArgs('\'%s\' is neither a build description nor a supported arch' % arg) + # take it as arch (even though this is no supported arch) - hopefully, this invalid + # arch will be detected below + arg_arch = arg else: arg_repository, arg_arch, arg_descr = args @@ -5457,41 +5483,41 @@ def parse_repoarchdescr(self, args, noinit = False, alternative_project = None, # store list of repos for potential offline use repolistfile = os.path.join(os.getcwd(), osc.core.store, "_build_repositories") if noinit: - if os.path.exists(repolistfile): - f = open(repolistfile, 'r') - repositories = [ l.strip()for l in f.readlines()] - f.close() + repositories = Repo.fromfile(repolistfile) else: project = alternative_project or store_read_project('.') apiurl = self.get_api_url() - repositories = get_repositories_of_project(apiurl, project) + repositories = list(get_repos_of_project(apiurl, project)) if not len(repositories): raise oscerr.WrongArgs('no repositories defined for project \'%s\'' % project) - try: - f = open(repolistfile, 'w') - f.write('\n'.join(repositories) + '\n') - f.close() - except: - pass + if alternative_project is None: + # only persist our own repos + Repo.tofile(repolistfile, repositories) - if not arg_repository and len(repositories): + repo_names = sorted(set([r.name for r in repositories])) + if not arg_repository and repositories: + # XXX: we should avoid hardcoding repository names # Use a default value from config, but just even if it's available - # unless try standard, or openSUSE_Factory - arg_repository = repositories[-1] - for repository in (conf.config['build_repository'], 'standard', 'openSUSE_Factory'): - if repository in repositories: + # unless try standard, or openSUSE_Factory, or openSUSE_Tumbleweed + arg_repository = repositories[-1].name + for repository in (conf.config['build_repository'], 'standard', 'openSUSE_Factory', 'openSUSE_Tumbleweed'): + if repository in repo_names: arg_repository = repository break if not arg_repository: raise oscerr.WrongArgs('please specify a repository') - elif noinit == False and not arg_repository in repositories: - raise oscerr.WrongArgs('%s is not a valid repository, use one of: %s' % (arg_repository, ', '.join(repositories))) + if not noinit: + if not arg_repository in repo_names: + raise oscerr.WrongArgs('%s is not a valid repository, use one of: %s' % (arg_repository, ', '.join(repo_names))) + arches = [r.arch for r in repositories if r.name == arg_repository and r.arch] + if arches and not arg_arch in arches: + raise oscerr.WrongArgs('%s is not a valid arch for the repository %s, use one of: %s' % (arg_arch, arg_repository, ', '.join(arches))) # can be implemented using # reduce(lambda x, y: x + y, (glob.glob(x) for x in ('*.spec', '*.dsc', '*.kiwi'))) # but be a bit more readable :) - descr = glob.glob('*.spec') + glob.glob('*.dsc') + glob.glob('*.kiwi') + glob.glob('PKGBUILD') + descr = glob.glob('*.spec') + glob.glob('*.dsc') + glob.glob('*.kiwi') + glob.glob('*.livebuild') + glob.glob('PKGBUILD') # FIXME: # * request repos from server and select by build type. @@ -5506,7 +5532,7 @@ def parse_repoarchdescr(self, args, noinit = False, alternative_project = None, pac = os.path.basename(os.getcwd()) if is_package_dir(os.getcwd()): pac = store_read_package(os.getcwd()) - extensions = ['spec', 'dsc', 'kiwi'] + extensions = ['spec', 'dsc', 'kiwi', 'livebuild'] cands = [i for i in descr for ext in extensions if i == '%s-%s.%s' % (pac, arg_repository, ext)] if len(cands) == 1: arg_descr = cands[0] @@ -5517,7 +5543,7 @@ def parse_repoarchdescr(self, args, noinit = False, alternative_project = None, if not arg_descr: msg = 'Multiple build description files found: %s' % ', '.join(descr) elif not ignore_descr: - msg = 'Missing argument: build description (spec, dsc or kiwi file)' + msg = 'Missing argument: build description (spec, dsc, kiwi or livebuild file)' try: p = Package('.') if p.islink() and not p.isexpanded(): @@ -5657,10 +5683,9 @@ def do_build(self, subcmd, opts, *args): import osc.build - if not os.path.exists('/usr/lib/build/debtransform') \ - and not os.path.exists('/usr/lib/lbuild/debtransform'): - sys.stderr.write('Error: you need build.rpm with version 2007.3.12 or newer.\n') - sys.stderr.write('See http://download.opensuse.org/repositories/openSUSE:/Tools/\n') + if which(conf.config['build-cmd']) is None: + print('Error: build (\'%s\') command not found' % conf.config['build-cmd'], file=sys.stderr) + print('Install the build package from http://download.opensuse.org/repositories/openSUSE:/Tools/', file=sys.stderr) return 1 if opts.debuginfo and opts.disable_debuginfo: @@ -5704,9 +5729,6 @@ def do_build(self, subcmd, opts, *args): if not os.path.isdir(d): raise oscerr.WrongOptions('Preferred package location \'%s\' is not a directory' % d) - if opts.noinit and opts.offline: - raise oscerr.WrongOptions('--noinit and --offline are mutually exclusive') - if opts.offline and opts.preload: raise oscerr.WrongOptions('--offline and --preload are mutually exclusive') @@ -5720,7 +5742,7 @@ def _do_rbuild(self, subcmd, opts, *args): # drop the --argument, value tuple from the list def drop_arg2(lst, name): - if not name: + if not name: return lst while name in lst: i = lst.index(name) @@ -5745,7 +5767,7 @@ def rsync_dirs_2host(hostargs, short_name, long_name, dirs): hostprefer = os.path.join( hostpath, basename, - "%s__" % (long_name.replace('-','_')), + "%s__" % (long_name.replace('-', '_')), os.path.basename(os.path.abspath(pdir))) hostargs.append(long_name) hostargs.append(hostprefer) @@ -5815,7 +5837,7 @@ def rsync_dirs_2host(hostargs, short_name, long_name, dirs): return ret for arg, long_name in ((opts.rsyncsrc, '--rsync-src'), (opts.overlay, '--overlay')): - if not arg: + if not arg: continue ret = rsync_dirs_2host(hostargs, None, long_name, (arg, )) if ret != 0: @@ -5857,8 +5879,10 @@ def rsync_dirs_2host(hostargs, short_name, long_name, dirs): help='specify the used build target project') @cmdln.option('--noinit', '--no-init', action='store_true', help='do not guess/verify specified repository') - @cmdln.option('-r', '--root', action='store_true', + @cmdln.option('-r', '--login-as-root', action='store_true', help='login as root instead of abuild') + @cmdln.option('--root', metavar='ROOT', + help='Path to the buildroot') @cmdln.option('-o', '--offline', action='store_true', help='Use cached data without contacting the api server') def do_chroot(self, subcmd, opts, *args): @@ -5882,17 +5906,20 @@ def do_chroot(self, subcmd, opts, *args): sys.exit(1) user = 'abuild' - if opts.root: + if opts.login_as_root: user = 'root' - repository, arch, descr = self.parse_repoarchdescr(args, opts.noinit or opts.offline, opts.alternative_project) - project = opts.alternative_project or store_read_project('.') - if opts.local_package: - package = os.path.splitext(descr)[0] - else: - package = store_read_package('.') - apihost = urlsplit(self.get_api_url())[1] - buildroot = os.environ.get('OSC_BUILD_ROOT', conf.config['build-root']) \ - % {'repo': repository, 'arch': arch, 'project': project, 'package': package, 'apihost': apihost} + buildroot = opts.root + if buildroot is None: + repository, arch, descr = self.parse_repoarchdescr(args, opts.noinit or opts.offline, opts.alternative_project) + project = opts.alternative_project or store_read_project('.') + if opts.local_package: + package = os.path.splitext(descr)[0] + else: + package = store_read_package('.') + apihost = urlsplit(self.get_api_url())[1] + if buildroot is None: + buildroot = os.environ.get('OSC_BUILD_ROOT', conf.config['build-root']) \ + % {'repo': repository, 'arch': arch, 'project': project, 'package': package, 'apihost': apihost} if not os.path.isdir(buildroot): raise oscerr.OscIOError(None, '\'%s\' is not a directory' % buildroot) @@ -6078,7 +6105,7 @@ def do_service(self, subcmd, opts, *args): osc service remoterun [PROJECT PACKAGE] COMMAND can be: - run r run defined services locally, it takes an optional parameter to run only a + run r run defined services locally, it takes an optional parameter to run only a specified source service. In case parameters exist for this one in _service file they are used. disabledrun dr run disabled or server side only services locally and store files as local created @@ -6125,7 +6152,7 @@ def do_service(self, subcmd, opts, *args): elif command == "disabledrun" or command == "dr": mode = "disabled" - p.run_source_services(mode, singleservice) + return p.run_source_services(mode, singleservice) @cmdln.option('-a', '--arch', metavar='ARCH', help='trigger rebuilds for a specific architecture') @@ -6396,7 +6423,7 @@ def do_getbinaries(self, subcmd, opts, *args): if package is None: package = meta_get_packagelist(apiurl, project) - else: + else: package = [package] # Set binary target directory and create if not existing @@ -6479,7 +6506,7 @@ def do_my(self, subcmd, opts, *args): """ # TODO: please clarify the difference between sr and rq. - # My first implementeation was to make no difference between requests FROM one + # My first implementeation was to make no difference between requests FROM one # of my projects and TO one of my projects. The current implementation appears to make this difference. # The usage above indicates, that sr would be a subset of rq, which is no the case with my tests. # jw. @@ -6573,11 +6600,11 @@ def do_my(self, subcmd, opts, *args): requests = [] # open reviews u = makeurl(apiurl, ['request'], { - 'view' : 'collection', + 'view': 'collection', 'states': 'review', 'reviewstates': 'new', 'roles': 'reviewer', - 'user' : user, + 'user': user, }) f = http_GET(u) root = ET.parse(f).getroot() @@ -6590,10 +6617,10 @@ def do_my(self, subcmd, opts, *args): print("") # open requests u = makeurl(apiurl, ['request'], { - 'view' : 'collection', + 'view': 'collection', 'states': 'new', 'roles': 'maintainer', - 'user' : user, + 'user': user, }) f = http_GET(u) root = ET.parse(f).getroot() @@ -6606,10 +6633,10 @@ def do_my(self, subcmd, opts, *args): print("") # declined requests submitted by me u = makeurl(apiurl, ['request'], { - 'view' : 'collection', + 'view': 'collection', 'states': 'declined', 'roles': 'creator', - 'user' : user, + 'user': user, }) f = http_GET(u) root = ET.parse(f).getroot() @@ -6686,7 +6713,7 @@ def do_my(self, subcmd, opts, *args): help='match only when given attribute exists in meta data') @cmdln.option('-v', '--verbose', action='store_true', help='show more information') - @cmdln.option('-V', '--version', action='store_true', + @cmdln.option('-V', '--version', action='store_true', help='show package version, revision, and srcmd5. CAUTION: This is slow and unreliable') @cmdln.option('-i', '--involved', action='store_true', help='show projects/packages where given person (or myself) is involved as bugowner or maintainer') @@ -6879,7 +6906,7 @@ def build_xpath(attr, what, substr = False): continue # construct a sorted, flat list # Sort by first column, follwed by second column if we have two columns, else sort by first. - results.sort(lambda x, y: ( cmp(x[0], y[0]) or + results.sort(lambda x, y: ( cmp(x[0], y[0]) or (len(x)>1 and len(y)>1 and cmp(x[1], y[1])) )) new = [] for i in results: @@ -7044,6 +7071,8 @@ def do_importsrcpkg(self, subcmd, opts, srpm): @cmdln.option('-X', '-m', '--method', default='GET', metavar='HTTP_METHOD', help='specify HTTP method to use (GET|PUT|DELETE|POST)') + @cmdln.option('-e', '--edit', default=None, action='store_true', + help='GET, edit and PUT the location') @cmdln.option('-d', '--data', default=None, metavar='STRING', help='specify string data for e.g. POST') @cmdln.option('-T', '-f', '--file', default=None, metavar='FILE', @@ -7064,6 +7093,7 @@ def do_api(self, subcmd, opts, url): Examples: osc api /source/home:user osc api -X PUT -T /etc/fstab source/home:user/test5/myfstab + osc api -e /configuration ${cmd_usage} ${cmd_option_list} @@ -7091,10 +7121,17 @@ def do_api(self, subcmd, opts, url): data=opts.data, file=opts.file, headers=opts.headers) - out = r.read() - sys.stdout.write(out) + if opts.edit: + text = edit_text(out) + r = http_request("PUT", + url, + data=text, + headers=opts.headers) + out = r.read() + + sys.stdout.write(out) @cmdln.option('-b', '--bugowner-only', action='store_true', @@ -7189,7 +7226,7 @@ def setBugownerHelper(apiurl, project, package, bugowner): roles = [ 'bugowner', 'maintainer' ] if len(opts.role): roles = opts.role - if opts.bugowner_only or opts.bugowner or subcmd == 'bugowner': + elif opts.bugowner_only or opts.bugowner or subcmd == 'bugowner': roles = [ 'bugowner' ] args = slash_split(args) @@ -7213,7 +7250,7 @@ def setBugownerHelper(apiurl, project, package, bugowner): apiurl = self.get_api_url() - # Try the OBS 2.4 way first. + # Try the OBS 2.4 way first. if binary or opts.user or opts.group: limit = None if opts.all: @@ -7224,19 +7261,24 @@ def setBugownerHelper(apiurl, project, package, bugowner): filterroles = None if binary: searchresult = owner(apiurl, binary, "binary", usefilter=filterroles, devel=None, limit=limit) - if not searchresult and (opts.set_bugowner or opts.set_bugowner_request): - # filtered search did not succeed, but maybe we want to set an owner initially? - searchresult = owner(apiurl, binary, "binary", usefilter="", devel=None, limit=-1) - if searchresult: - print("WARNING: the binary exists, but has no matching maintainership roles defined.") - print("Do you want to set it in the container where the binary appeared first?") - result = searchresult.find('owner') - print("This is: " + result.get('project'), end=' ') - if result.get('package'): - print (" / " + result.get('package')) - repl = raw_input('\nUse this container? (y/n) ') - if repl.lower() != 'y': - searchresult = None + if searchresult != None and len(searchresult) == 0: + # We talk to an OBS 2.4 or later understanding the call + if opts.set_bugowner or opts.set_bugowner_request: + # filtered search did not succeed, but maybe we want to set an owner initially? + searchresult = owner(apiurl, binary, "binary", usefilter="", devel=None, limit=-1) + if searchresult: + print("WARNING: the binary exists, but has no matching maintainership roles defined.") + print("Do you want to set it in the container where the binary appeared first?") + result = searchresult.find('owner') + print("This is: " + result.get('project'), end=' ') + if result.get('package'): + print (" / " + result.get('package')) + repl = raw_input('\nUse this container? (y/n) ') + if repl.lower() != 'y': + searchresult = None + else: + print("Empty search result, you may want to search with other or all roles via -r ''") + return elif opts.user: searchresult = owner(apiurl, opts.user, "user", usefilter=filterroles, devel=None) elif opts.group: @@ -7361,7 +7403,7 @@ def setBugownerHelper(apiurl, project, package, bugowner): else: print("Defined in project: ", definingproject) - if prj: + if prj: # not for user/group search for role in roles: if opts.bugowner and not len(maintainers.get(role, [])): @@ -7951,18 +7993,16 @@ def do_vc(self, subcmd, opts, *args): except IndexError: pass + cmd_list = [conf.config['vc-cmd']] if meego_style: if not os.path.exists('/usr/bin/vc'): print('Error: you need meego-packaging-tools for /usr/bin/vc command', file=sys.stderr) return 1 cmd_list = ['/usr/bin/vc'] - else: - if not os.path.exists('/usr/lib/build/vc'): - print('Error: you need build.rpm with version 2009.04.17 or newer', file=sys.stderr) - print('See http://download.opensuse.org/repositories/openSUSE:/Tools/', file=sys.stderr) - return 1 - - cmd_list = ['/usr/lib/build/vc'] + elif which(cmd_list[0]) is None: + print('Error: vc (\'%s\') command not found' % cmd_list[0], file=sys.stderr) + print('Install the build package from http://download.opensuse.org/repositories/openSUSE:/Tools/', file=sys.stderr) + return 1 # set user's email if no mailaddr exists if 'mailaddr' not in os.environ: diff --git a/osc/conf.py b/osc/conf.py index 1ff8c08..3239cd8 100644 --- a/osc/conf.py +++ b/osc/conf.py @@ -112,6 +112,8 @@ def _get_processors(): 'build-vmdisk-rootsize': '', # optional for VM builds 'build-vmdisk-swapsize': '', # optional for VM builds 'build-vmdisk-filesystem': '', # optional for VM builds + 'build-kernel': '', # optional for VM builds + 'build-initrd': '', # optional for VM builds 'build-jobs': _get_processors(), 'builtin_signature_check': '1', # by default use builtin check for verify pkgs @@ -172,6 +174,8 @@ def _get_processors(): 'maintenance_attribute': 'OBS:MaintenanceProject', 'maintained_update_project_attribute': 'OBS:UpdateProject', 'show_download_progress': '0', + # path to the vc script + 'vc-cmd': '/usr/lib/build/vc' } # being global to this module, this dict can be accessed from outside @@ -225,6 +229,12 @@ def _get_processors(): # e.g. /var/tmp/FILE.swap #build-swap = /var/tmp/FILE.swap +# build-kernel is the boot kernel used for VM builds +#build-kernel = /boot/vmlinuz + +# build-initrd is the boot initrd used for VM builds +#build-initrd = /boot/initrd + # build-memory is the amount of memory used in the VM # value in MB - e.g. 512 #build-memory = 512 @@ -466,7 +476,7 @@ def http_error_404(self, *args): return None authhandler_class = OscHTTPBasicAuthHandler - elif sys.version_info >= (2, 6, 6) and sys.version_info < (2, 7, 99): + elif sys.version_info >= (2, 6, 6) and sys.version_info < (2, 7, 1): class OscHTTPBasicAuthHandler(HTTPBasicAuthHandler): def http_error_404(self, *args): self.reset_retry_count() @@ -511,10 +521,10 @@ def retry_http_basic_auth(self, host, req, realm): capath = i break if not cafile and not capath: - raise Exception('No CA certificates found') + raise oscerr.OscIOError(None, 'No CA certificates found') ctx = oscssl.mySSLContext() if ctx.load_verify_locations(capath=capath, cafile=cafile) != 1: - raise Exception('No CA certificates found') + raise oscerr.OscIOError(None, 'No CA certificates found') opener = m2urllib2.build_opener(ctx, oscssl.myHTTPSHandler(ssl_context=ctx, appname='osc'), HTTPCookieProcessor(cookiejar), authhandler, proxyhandler) else: print("WARNING: SSL certificate checks disabled. Connection is insecure!\n", file=sys.stderr) @@ -923,6 +933,8 @@ def get_config(override_conffile=None, api_host_options[apiurl][key] = cp.getboolean(url, key) else: api_host_options[apiurl][key] = cp.get(url, key) + if cp.has_option(url, 'build-root', proper=True): + api_host_options[apiurl]['build-root'] = cp.get(url, 'build-root', raw=True) if not 'sslcertck' in api_host_options[apiurl]: api_host_options[apiurl]['sslcertck'] = True diff --git a/osc/core.py b/osc/core.py index a6ee49c..17d21e3 100644 --- a/osc/core.py +++ b/osc/core.py @@ -5,7 +5,7 @@ from __future__ import print_function -__version__ = '0.145git' +__version__ = '0.151' # __store_version__ is to be incremented when the format of the working copy # "store" changes in an incompatible way. Please add any needed migration @@ -273,18 +273,22 @@ def read(self, serviceinfo_node, append=False): for service in services: name = service.get('name') + if len(name) < 3 or '/' in name: + raise oscerr.APIError("invalid service name") mode = service.get('mode', None) data = { 'name' : name, 'mode' : '' } if mode: data['mode'] = mode try: + command = [ name ] for param in service.findall('param'): option = param.get('name', None) value = "" if param.text: value = param.text - name += " --" + option + " '" + value + "'" - data['command'] = name + command.append("--"+option) + command.append(value) + data['command'] = command self.services.append(data) except: msg = 'invalid service format:\n%s' % ET.tostring(serviceinfo_node, encoding=ET_ENCODING) @@ -335,6 +339,12 @@ def addDownloadUrl(self, serviceinfo_node, url_string): r.append( s ) return r + def addSetVersion(self, serviceinfo_node): + r = serviceinfo_node + s = ET.Element( "service", name="set_version" ) + r.append( s ) + return r + def addGitUrl(self, serviceinfo_node, url_string): r = serviceinfo_node s = ET.Element( "service", name="tar_scm" ) @@ -347,7 +357,7 @@ def addRecompressTar(self, serviceinfo_node): r = serviceinfo_node s = ET.Element( "service", name="recompress" ) ET.SubElement(s, "param", name="file").text = "*.tar" - ET.SubElement(s, "param", name="compression").text = "bz2" + ET.SubElement(s, "param", name="compression").text = "xz" r.append( s ) return r @@ -366,7 +376,7 @@ def execute(self, dir, callmode = None, singleservice = None, verbose = None): allservices = self.services or [] if singleservice and not singleservice in allservices: # set array to the manual specified singleservice, if it is not part of _service file - data = { 'name' : singleservice, 'command' : singleservice, 'mode' : '' } + data = { 'name' : singleservice, 'command' : [ singleservice ], 'mode' : '' } allservices = [data] # set environment when using OBS 2.3 or later @@ -387,21 +397,21 @@ def execute(self, dir, callmode = None, singleservice = None, verbose = None): continue if service['mode'] != "trylocal" and service['mode'] != "localonly" and callmode == "trylocal": continue - call = service['command'] temp_dir = None try: temp_dir = tempfile.mkdtemp() - name = call.split(None, 1)[0] - if not os.path.exists("/usr/lib/obs/service/"+name): - raise oscerr.PackageNotInstalled("obs-service-"+name) - cmd = "/usr/lib/obs/service/" + call + " --outdir " + temp_dir + cmd = service['command'] + if not os.path.exists("/usr/lib/obs/service/"+cmd[0]): + raise oscerr.PackageNotInstalled("obs-service-%s"%cmd[0]) + cmd[0] = "/usr/lib/obs/service/"+cmd[0] + cmd = cmd + [ "--outdir", temp_dir ] if conf.config['verbose'] > 1 or verbose: - print("Run source service:", cmd) - r = run_external(cmd, shell=True) + print("Run source service:", ' '.join(cmd)) + r = run_external(*cmd) if r != 0: - print("Aborting: service call failed: " + cmd) - # FIXME: addDownloadUrlService calls si.execute after + print("Aborting: service call failed: ", ' '.join(cmd)) + # FIXME: addDownloadUrlService calls si.execute after # updating _services. return r @@ -409,6 +419,7 @@ def execute(self, dir, callmode = None, singleservice = None, verbose = None): for filename in os.listdir(temp_dir): shutil.move( os.path.join(temp_dir, filename), os.path.join(dir, filename) ) else: + name = service['name'] for filename in os.listdir(temp_dir): shutil.move( os.path.join(temp_dir, filename), os.path.join(dir, "_service:"+name+":"+filename) ) finally: @@ -480,6 +491,31 @@ def __str__(self): else: return 'None' +class DirectoryServiceinfo: + def __init__(self): + self.code = None + self.xsrcmd5 = None + self.lsrcmd5 = None + self.error = '' + + def read(self, serviceinfo_node): + if serviceinfo_node is None: + return + self.code = serviceinfo_node.get('code') + self.xsrcmd5 = serviceinfo_node.get('xsrcmd5') + self.lsrcmd5 = serviceinfo_node.get('lsrcmd5') + self.error = serviceinfo_node.find('error') + if self.error: + self.error = self.error.text + + def isexpanded(self): + """ + Returns true, if the directory contains the "expanded"/generated service files + """ + return self.lsrcmd5 is not None and self.xsrcmd5 is None + + def haserror(self): + return self.error is not None # http://effbot.org/zone/element-lib.htm#prettyprint def xmlindent(elem, level=0): @@ -548,8 +584,6 @@ class Project: """ REQ_STOREFILES = ('_project', '_apiurl') - if conf.config['do_package_tracking']: - REQ_STOREFILES += ('_packages',) def __init__(self, dir, getPackageList=True, progress_obj=None, wc_check=True): """ @@ -610,7 +644,10 @@ def __init__(self, dir, getPackageList=True, progress_obj=None, wc_check=True): def wc_check(self): global store dirty_files = [] - for fname in Project.REQ_STOREFILES: + req_storefiles = Project.REQ_STOREFILES + if conf.config['do_package_tracking']: + req_storefiles += ('_packages',) + for fname in req_storefiles: if not os.path.exists(os.path.join(self.absdir, store, fname)): dirty_files.append(fname) return dirty_files @@ -795,6 +832,8 @@ def update(self, pacs = (), expand_link=False, unexpand_link=False, service_file # update complete project # packages which no longer exists upstream upstream_del = [ pac for pac in self.pacs_have if not pac in self.pacs_available and self.get_state(pac) != 'A'] + sinfo_pacs = [pac for pac in self.pacs_have if self.get_state(pac) in (' ', 'D') and not pac in self.pacs_broken] + sinfos = get_project_sourceinfo(self.apiurl, self.name, True, *sinfo_pacs) for pac in upstream_del: if self.status(pac) != '!': @@ -813,12 +852,13 @@ def update(self, pacs = (), expand_link=False, unexpand_link=False, service_file if pac in self.pacs_broken: if self.get_state(pac) != 'A': checkout_package(self.apiurl, self.name, pac, - pathname=getTransActPath(os.path.join(self.dir, pac)), prj_obj=self, \ + pathname=getTransActPath(os.path.join(self.dir, pac)), prj_obj=self, prj_dir=self.dir, expand_link=not unexpand_link, progress_obj=self.progress_obj) elif state == ' ': # do a simple update p = Package(os.path.join(self.dir, pac), progress_obj=self.progress_obj) rev = None + needs_update = True if expand_link and p.islink() and not p.isexpanded(): if p.haslinkerror(): try: @@ -833,19 +873,29 @@ def update(self, pacs = (), expand_link=False, unexpand_link=False, service_file rev = p.linkinfo.lsrcmd5 print('Unexpanding to rev', rev) elif p.islink() and p.isexpanded(): - rev = p.latest_rev() + needs_update = p.update_needed(sinfos[p.name]) + if needs_update: + rev = p.latest_rev() + elif p.hasserviceinfo() and p.serviceinfo.isexpanded() and not service_files: + # FIXME: currently, do_update does not propagate the --server-side-source-service-files + # option to this method. Consequence: an expanded service is always unexpanded during + # an update (TODO: discuss if this is a reasonable behavior (at least this the default + # behavior for a while)) + needs_update = True + else: + needs_update = p.update_needed(sinfos[p.name]) print('Updating %s' % p.name) - p.update(rev, service_files) + if needs_update: + p.update(rev, service_files) + else: + print('At revision %s.' % p.rev) if unexpand_link: p.unmark_frozen() elif state == 'D': - # TODO: Package::update has to fixed to behave like svn does - if pac in self.pacs_broken: - checkout_package(self.apiurl, self.name, pac, - pathname=getTransActPath(os.path.join(self.dir, pac)), prj_obj=self, \ - prj_dir=self.dir, expand_link=expand_link, progress_obj=self.progress_obj) - else: - Package(os.path.join(self.dir, pac), progress_obj=self.progress_obj).update() + # pac exists (the non-existent pac case was handled in the first if block) + p = Package(os.path.join(self.dir, pac), progress_obj=self.progress_obj) + if p.update_needed(sinfos[p.name]): + p.update() elif state == 'A' and pac in self.pacs_available: # file/dir called pac already exists and is under version control msg = 'can\'t add package \'%s\': Object already exists' % pac @@ -1593,6 +1643,8 @@ def update_datastructs(self): self.linkinfo = Linkinfo() self.linkinfo.read(files_tree_root.find('linkinfo')) + self.serviceinfo = DirectoryServiceinfo() + self.serviceinfo.read(files_tree_root.find('serviceinfo')) self.filenamelist = [] self.filelist = [] @@ -1677,6 +1729,12 @@ def linkerror(self): """ return self.linkinfo.error + def hasserviceinfo(self): + """ + Returns True, if this package contains services. + """ + return self.serviceinfo.lsrcmd5 is not None or self.serviceinfo.xsrcmd5 is not None + def update_local_pacmeta(self): """ Update the local _meta file in the store. @@ -1976,7 +2034,7 @@ def update_package_meta(self, force=False): def mark_frozen(self): store_write_string(self.absdir, '_frozenlink', '') print() - print("The link in this package is currently broken. Checking") + print("The link in this package (\"%s\") is currently broken. Checking" % self.name) print("out the last working version instead; please use 'osc pull'") print("to merge the conflicts.") print() @@ -2043,6 +2101,42 @@ def __get_rev_changes(self, revfiles): return kept, added, deleted, services + def update_needed(self, sinfo): + # this method might return a false-positive (that is a True is returned, + # even though no update is needed) (for details, see comments below) + if self.islink(): + if self.isexpanded(): + # check if both revs point to the same expanded sources + # Note: if the package contains a _service file, sinfo.srcmd5's lsrcmd5 + # points to the "expanded" services (xservicemd5) => chances + # for a false-positive are high, because osc usually works on the + # "unexpanded" services. + # Once the srcserver supports something like noservice=1, we can get rid of + # this false-positives (patch was already sent to the ml) (but this also + # requires some slight changes in osc) + return sinfo.get('srcmd5') != self.srcmd5 + elif self.hasserviceinfo(): + # check if we have expanded or unexpanded services + if self.serviceinfo.isexpanded(): + return sinfo.get('lsrcmd5') != self.srcmd5 + else: + # again, we might have a false-positive here, because + # a mismatch of the "xservicemd5"s does not neccessarily + # imply a change in the "unexpanded" services. + return sinfo.get('lsrcmd5') != self.serviceinfo.xsrcmd5 + # simple case: unexpanded sources and no services + # self.srcmd5 should also work + return sinfo.get('lsrcmd5') != self.linkinfo.lsrcmd5 + elif self.hasserviceinfo(): + if self.serviceinfo.isexpanded(): + return sinfo.get('srcmd5') != self.srcmd5 + else: + # cannot handle this case, because the sourceinfo does not contain + # information about the lservicemd5. Once the srcserver supports + # a noservice=1 query parameter, we can handle this case. + return True + return sinfo.get('srcmd5') != self.srcmd5 + def update(self, rev = None, service_files = False, size_limit = None): import tempfile rfiles = [] @@ -2269,6 +2363,10 @@ def get_comment(self): """return data from tag""" raise NotImplementedError() + def get_description(self): + """return data from tag""" + raise NotImplementedError() + def to_xml(self): """serialize object to XML""" root = ET.Element(self.get_node_name()) @@ -2276,6 +2374,8 @@ def to_xml(self): val = getattr(self, attr) if not val is None: root.set(attr, val) + if self.get_description(): + ET.SubElement(root, 'description').text = self.get_description() if self.get_comment(): ET.SubElement(root, 'comment').text = self.get_comment() return root @@ -2312,6 +2412,50 @@ def get_node_attrs(self): def get_comment(self): return self.comment + def get_description(self): + return None + + +class RequestHistory(AbstractState): + """Represents a history element of a request""" + re_name = re.compile(r'^Request (?:got )?([^\s]+)$') + + def __init__(self, history_node): + AbstractState.__init__(self, history_node.tag) + self.who = history_node.get('who') + self.when = history_node.get('when') + if not history_node.find('description') is None and \ + history_node.find('description').text: + # OBS 2.6 + self.description = history_node.find('description').text.strip() + else: + # OBS 2.5 and before + self.description = history_node.get('name') + self.comment = '' + if not history_node.find('comment') is None and \ + history_node.find('comment').text: + self.comment = history_node.find('comment').text.strip() + self.name = self._parse_name(history_node) + + def _parse_name(self, history_node): + name = history_node.get('name', None) + if name is not None: + # OBS 2.5 and before + return name + mo = self.re_name.search(self.description) + if mo is not None: + return mo.group(1) + return self.description + + def get_node_attrs(self): + return ('who', 'when') + + def get_description(self): + return self.description + + def get_comment(self): + return self.comment + class RequestState(AbstractState): """Represents the state of a request""" @@ -2323,6 +2467,9 @@ def __init__(self, state_node): self.name = state_node.get('name') self.who = state_node.get('who') self.when = state_node.get('when') + if state_node.find('description') is None: + # OBS 2.6 has it always, before it did not exist + self.description = state_node.get('description') self.comment = '' if not state_node.find('comment') is None and \ state_node.find('comment').text: @@ -2334,6 +2481,9 @@ def get_node_attrs(self): def get_comment(self): return self.comment + def get_description(self): + return None + class Action: """ @@ -2361,13 +2511,13 @@ class Action: # allowed types + the corresponding (allowed) attributes type_args = {'submit': ('src_project', 'src_package', 'src_rev', 'tgt_project', 'tgt_package', 'opt_sourceupdate', 'acceptinfo_rev', 'acceptinfo_srcmd5', 'acceptinfo_xsrcmd5', 'acceptinfo_osrcmd5', - 'acceptinfo_oxsrcmd5', 'opt_updatelink'), + 'acceptinfo_oxsrcmd5', 'opt_updatelink', 'opt_makeoriginolder'), 'add_role': ('tgt_project', 'tgt_package', 'person_name', 'person_role', 'group_name', 'group_role'), 'set_bugowner': ('tgt_project', 'tgt_package', 'person_name', 'group_name'), 'maintenance_release': ('src_project', 'src_package', 'src_rev', 'tgt_project', 'tgt_package', 'person_name', 'acceptinfo_rev', 'acceptinfo_srcmd5', 'acceptinfo_xsrcmd5', 'acceptinfo_osrcmd5', 'acceptinfo_oxsrcmd5'), - 'maintenance_incident': ('src_project', 'src_package', 'src_rev', 'tgt_project', 'tgt_releaseproject', 'person_name', 'opt_sourceupdate'), + 'maintenance_incident': ('src_project', 'src_package', 'src_rev', 'tgt_project', 'tgt_releaseproject', 'person_name', 'opt_sourceupdate', 'opt_makeoriginolder'), 'delete': ('tgt_project', 'tgt_package', 'tgt_repository'), 'change_devel': ('src_project', 'src_package', 'tgt_project', 'tgt_package'), 'group': ('grouped_id', )} @@ -2465,6 +2615,7 @@ def _init_attributes(self): self.reqid = None self.title = '' self.description = '' + self.priority = None self.state = None self.accept_at = None self.actions = [] @@ -2490,8 +2641,10 @@ def read(self, root): self.actions.append(Action.from_xml(action)) for review in root.findall('review'): self.reviews.append(ReviewState(review)) - for hist_state in root.findall('history'): - self.statehistory.append(RequestState(hist_state)) + for history_element in root.findall('history'): + self.statehistory.append(RequestHistory(history_element)) + if not root.find('priority') is None and root.find('priority').text: + self.priority = root.find('priority').text.strip() if not root.find('accept_at') is None and root.find('accept_at').text: self.accept_at = root.find('accept_at').text.strip() if not root.find('title') is None: @@ -2537,6 +2690,8 @@ def to_xml(self): ET.SubElement(root, 'description').text = self.description if self.accept_at: ET.SubElement(root, 'accept_at').text = self.accept_at + if self.priority: + ET.SubElement(root, 'priority').text = self.priority return root def to_str(self): @@ -2629,6 +2784,10 @@ def prj_pkg_join(prj, pkg, repository=None): if action.src_package == action.tgt_package: tgt_package = '' d['target'] = prj_pkg_join(action.tgt_project, tgt_package) + if action.opt_makeoriginolder: + d['target'] = d['target'] + ' ***make origin older***' + if action.opt_updatelink: + d['target'] = d['target'] + ' ***update link***' elif action.type == 'add_role': roles = [] if action.person_name and action.person_role: @@ -2663,7 +2822,7 @@ def list_view(self): tmpl = ' Review by %(type)-10s is %(state)-10s %(by)-50s' for review in self.reviews: lines.append(tmpl % Request.format_review(review)) - history = ['%s(%s)' % (hist.name, hist.who) for hist in self.statehistory] + history = ['%s: %s' % (hist.description, hist.who) for hist in self.statehistory] if history: lines.append(' From: %s' % ' -> '.join(history)) if self.description: @@ -2678,6 +2837,8 @@ def __str__(self): lines = ['Request: #%s\n' % self.reqid] if self.accept_at and self.state.name in [ 'new', 'review' ]: lines.append(' *** This request will get automatically accepted after '+self.accept_at+' ! ***\n') + if self.priority in [ 'critical', 'important' ] and self.state.name in [ 'new', 'review' ]: + lines.append(' *** This request has classified as '+self.priority+' ! ***\n') for action in self.actions: tmpl = ' %(type)-13s %(source)s %(target)s' @@ -2704,7 +2865,7 @@ def __str__(self): if review.by_group: d['by'] = "Group: " + review.by_group if review.by_package: - d['by'] = "Package: " + review.by_project + "/" + review.by_package + d['by'] = "Package: " + review.by_project + "/" + review.by_package elif review.by_project: d['by'] = "Project: " + review.by_project d['when'] = review.when or '' @@ -2714,11 +2875,10 @@ def __str__(self): if reviews: lines.append('\nReview: %s' % indent.join(reviews)) - tmpl = '%(name)-10s %(when)-12s %(who)s' + tmpl = '%(when)-10s %(who)-12s %(desc)s' histories = [] for hist in reversed(self.statehistory): - d = {'name': hist.name, 'when': hist.when, - 'who': hist.who} + d = {'when': hist.when, 'who': hist.who, 'desc': hist.description} histories.append(tmpl % d) if histories: lines.append('\nHistory: %s' % indent.join(histories)) @@ -2745,11 +2905,11 @@ def shorttime(t): """ import time - if time.localtime()[0] == time.localtime(t)[0]: + if time.gmtime()[0] == time.gmtime(t)[0]: # same year - return time.strftime('%b %d %H:%M', time.localtime(t)) + return time.strftime('%b %d %H:%M %Z', time.gmtime(t)) else: - return time.strftime('%b %d %Y', time.localtime(t)) + return time.strftime('%b %d %Y', time.gmtime(t)) def is_project_dir(d): @@ -3091,11 +3251,13 @@ def check_store_version(dir): raise oscerr.WorkingCopyWrongVersion(msg) -def meta_get_packagelist(apiurl, prj, deleted=None): +def meta_get_packagelist(apiurl, prj, deleted=None, expand=False): query = {} if deleted: query['deleted'] = 1 + if expand: + query['expand'] = 1 u = makeurl(apiurl, ['source', prj], query) f = http_GET(u) @@ -3317,8 +3479,8 @@ def edit(self): break except HTTPError as e: error_help = "%d" % e.code - if e.headers.get('X-Opensuse-Errorcode'): - error_help = "%s (%d)" % (e.headers.get('X-Opensuse-Errorcode'), e.code) + if e.hdrs.get('X-Opensuse-Errorcode'): + error_help = "%s (%d)" % (e.hdrs.get('X-Opensuse-Errorcode'), e.code) print('BuildService API error:', error_help, file=sys.stderr) # examine the error - we can't raise an exception because we might want @@ -3503,6 +3665,37 @@ def show_upstream_xsrcmd5(apiurl, prj, pac, revision=None, linkrev=None, linkrep return li.xsrcmd5 +def show_project_sourceinfo(apiurl, project, nofilename, *packages): + query = ['view=info'] + if packages: + query.extend(['package=%s' % quote_plus(p) for p in packages]) + if nofilename: + query.append('nofilename=1') + f = http_GET(makeurl(apiurl, ['source', project], query=query)) + return f.read() + + +def get_project_sourceinfo(apiurl, project, nofilename, *packages): + try: + si = show_project_sourceinfo(apiurl, project, nofilename, *packages) + except HTTPError, e: + if e.code != 414: + raise + if len(packages) == 1: + raise oscerr.APIError('package name too long: %s' % packages[0]) + n = len(packages) / 2 + pkgs = packages[:n] + res = get_project_sourceinfo(apiurl, project, nofilename, *pkgs) + pkgs = packages[n:] + res.update(get_project_sourceinfo(apiurl, project, nofilename, *pkgs)) + return res + root = ET.fromstring(si) + res = {} + for sinfo in root.findall('sourceinfo'): + res[sinfo.get('package')] = sinfo + return res + + def show_upstream_rev_vrev(apiurl, prj, pac, revision=None, expand=False, meta=False): m = show_files_meta(apiurl, prj, pac, revision=revision, expand=expand, meta=meta) et = ET.fromstring(''.join(m)) @@ -3658,7 +3851,6 @@ def _edit_message_open_editor(filename, data, orig_mtime): return os.stat(filename).st_mtime != orig_mtime def edit_message(footer='', template='', templatelen=30): - import tempfile delim = '--This line, and those below, will be ignored--\n' data = '' if template != '': @@ -3668,18 +3860,24 @@ def edit_message(footer='', template='', templatelen=30): if lines[templatelen:]: footer = '%s\n\n%s' % ('\n'.join(lines[templatelen:]), footer) data += '\n' + delim + '\n' + footer + return edit_text(data, delim, suffix='.diff', template=template) + +def edit_text(data='', delim=None, suffix='.txt', template=''): + import tempfile try: - (fd, filename) = tempfile.mkstemp(prefix='osc-commitmsg', suffix='.diff') + (fd, filename) = tempfile.mkstemp(prefix='osc-editor', suffix=suffix) os.close(fd) mtime = os.stat(filename).st_mtime while True: file_changed = _edit_message_open_editor(filename, data, mtime) - msg = open(filename).read().split(delim)[0].rstrip() + msg = open(filename).read() + if delim: + msg = msg.split(delim)[0].rstrip() if msg and file_changed: break else: reason = 'Log message not specified' - if template and template == msg: + if template == msg: reason = 'Default log message was not changed. Press \'c\' to continue.' ri = raw_input('%s\na)bort, c)ontinue, e)dit: ' % reason) if ri in 'aA': @@ -3730,19 +3928,23 @@ def create_maintenance_request(apiurl, src_project, src_packages, tgt_project, t r.create(apiurl, addrevision=True) return r -# This creates an old style submit request for server api 1.0 def create_submit_request(apiurl, src_project, src_package=None, dst_project=None, dst_package=None, - message="", orev=None, src_update=None): + message="", orev=None, src_update=None, dst_updatelink=None): import cgi options_block = "" package = "" if src_package: package = """package="%s" """ % (src_package) + options_block = "" if src_update: - options_block = """%s """ % (src_update) + options_block += """%s""" % (src_update) + if dst_updatelink: + options_block += """true""" + options_block += "" + # Yes, this kind of xml construction is horrible targetxml = "" @@ -3753,12 +3955,12 @@ def create_submit_request(apiurl, targetxml = """ """ % ( dst_project, packagexml ) # XXX: keep the old template for now in order to work with old obs instances xml = """\ - - + + %s %s - + %s @@ -3781,14 +3983,20 @@ def create_submit_request(apiurl, root = ET.parse(f).getroot() r = root.get('id') except HTTPError as e: - if e.headers.get('X-Opensuse-Errorcode') == "submit_request_rejected": + if e.hdrs.get('X-Opensuse-Errorcode') == "submit_request_rejected": print("WARNING:") print("WARNING: Project does not accept submit request, request to open a NEW maintenance incident instead") print("WARNING:") - xpath = 'maintenance/maintains/@project = \'%s\'' % dst_project + xpath = 'maintenance/maintains/@project = \'%s\' and attribute/@name = \'%s\'' % (dst_project, conf.config['maintenance_attribute']) res = search(apiurl, project_id=xpath) root = res['project_id'] project = root.find('project') + if project is None: + print("WARNING: This project is not maintained in the maintenance project specified by '%s', looking elsewhere" % conf.config['maintenance_attribute']) + xpath = 'maintenance/maintains/@project = \'%s\'' % dst_project + res = search(apiurl, project_id=xpath) + root = res['project_id'] + project = root.find('project') if project is None: raise oscerr.APIError("Server did not define a default maintenance project, can't submit.") tproject = project.get('name') @@ -3800,7 +4008,7 @@ def create_submit_request(apiurl, def get_request(apiurl, reqid): - u = makeurl(apiurl, ['request', reqid]) + u = makeurl(apiurl, ['request', reqid], {'withfullhistory': '1'}) f = http_GET(u) root = ET.parse(f).getroot() @@ -3849,7 +4057,7 @@ def change_request_state_template(req, newstate): action = req.actions[0] tmpl_name = '%srequest_%s_template' % (action.type, newstate) tmpl = conf.config.get(tmpl_name, '') - tmpl = tmpl.replace('\\t', '\t').replace('\\n', '\n') + tmpl = tmpl.replace('\\t', '\t').replace('\\n', '\n') data = {'reqid': req.reqid, 'type': action.type, 'who': req.get_creator()} if req.actions[0].type == 'submit': data.update({'src_project': action.src_project, @@ -3862,7 +4070,7 @@ def change_request_state_template(req, newstate): print('error: cannot interpolate \'%s\' in \'%s\'' % (e.args[0], tmpl_name), file=sys.stderr) return '' -def get_review_list(apiurl, project='', package='', byuser='', bygroup='', byproject='', bypackage='', states=('new')): +def get_review_list(apiurl, project='', package='', byuser='', bygroup='', byproject='', bypackage='', states=()): # this is so ugly... def build_by(xpath, val): if 'all' in states: @@ -3873,16 +4081,16 @@ def build_by(xpath, val): s_xp = xpath_join(s_xp, '@state=\'%s\'' % state, inner=True) val = val.strip('[').strip(']') return xpath_join(xpath, 'review[%s and (%s)]' % (val, s_xp), op='and') + else: + # default case + return xpath_join(xpath, 'review[%s and @state=\'new\']' % val, op='and') return '' xpath = '' - xpath = xpath_join(xpath, 'state/@name=\'review\'', inner=True) - if not 'all' in states: - for state in states: - xpath = xpath_join(xpath, 'review/@state=\'%s\'' % state, inner=True) - if byuser or bygroup or bypackage or byproject: - # discard constructed xpath... - xpath = xpath_join('', 'state/@name=\'review\'', inner=True) + if states == (): + # default: requests which are still open and have reviews in "new" state + xpath = xpath_join('', 'state/@name=\'review\'', op='and') + xpath = xpath_join(xpath, 'review/@state=\'new\'', op='and') if byuser: xpath = build_by(xpath, '@by_user=\'%s\'' % byuser) if bygroup: @@ -4050,7 +4258,7 @@ def check_existing_requests(apiurl, src_project, src_package, dst_project, reqs = get_exact_request_list(apiurl, src_project, dst_project, src_package, dst_package, req_type='submit', - req_state=['new','review', 'declined']) + req_state=['new', 'review', 'declined']) repl = '' if reqs: print('There are already the following submit request: %s.' % \ @@ -4117,7 +4325,7 @@ def download(url, filename, progress_obj = None, mtime = None): try: o = os.fdopen(fd, 'wb') for buf in streamfile(url, http_GET, BUFSIZE, progress_obj=progress_obj): - o.write(bytes(buf,"utf-8")) + o.write(bytes(buf, "utf-8")) o.close() os.rename(tmpfile, filename) except: @@ -4435,13 +4643,13 @@ def checkout_package(apiurl, project, package, # if we are in a package dir, goto parent. # Hmm, with 'checkout_no_colon' in effect, we have directory levels that # do not easily reveal the fact, that they are part of a project path. - # At least this test should find that the parent of 'home/username/branches' + # At least this test should find that the parent of 'home/username/branches' # is a project (hack alert). Also goto parent in this case. root_dots = "../" elif is_project_dir("../.."): # testing two levels is better than one. - # May happen in case of checkout_no_colon, or - # if project roots were previously inconsistent + # May happen in case of checkout_no_colon, or + # if project roots were previously inconsistent root_dots = "../../" if is_project_dir(root_dots): if conf.config['checkout_no_colon']: @@ -4983,12 +5191,12 @@ def get_distibutions(apiurl, discon=False): for node in root.findall('entry'): if node.get('name').startswith('DISCONTINUED:'): rmap = {} - rmap['name'] = node.get('name').replace('DISCONTINUED:','').replace(':', ' ') + rmap['name'] = node.get('name').replace('DISCONTINUED:', '').replace(':', ' ') rmap['project'] = node.get('name') r.append (result_line_templ % rmap) - r.insert(0,'distribution project') - r.insert(1,'------------ -------') + r.insert(0, 'distribution project') + r.insert(1, '------------ -------') else: result_line_templ = '%(name)-25s %(project)-25s %(repository)-25s %(reponame)s' @@ -5007,8 +5215,8 @@ def get_distibutions(apiurl, discon=False): rmap['reponame'] = node5.text r.append(result_line_templ % rmap) - r.insert(0,'distribution project repository reponame') - r.insert(1,'------------ ------- ---------- --------') + r.insert(0, 'distribution project repository reponame') + r.insert(1, '------------ ------- ---------- --------') return r @@ -5035,6 +5243,30 @@ def __init__(self, name, arch): def __str__(self): return self.repo_line_templ % (self.name, self.arch) + def __repr__(self): + return 'Repo(%s %s)' % (self.name, self.arch) + + @staticmethod + def fromfile(filename): + if not os.path.exists(filename): + return [] + repos = [] + lines = open(filename, 'r').readlines() + for line in lines: + data = line.split() + if len(data) == 2: + repos.append(Repo(data[0], data[1])) + elif len(data) == 1: + # only for backward compatibility + repos.append(Repo(data[0], '')) + return repos + + @staticmethod + def tofile(filename, repos): + with open(filename, 'w') as f: + for repo in repos: + f.write('%s %s\n' % (repo.name, repo.arch)) + def get_repos_of_project(apiurl, prj): f = show_project_meta(apiurl, prj) root = ET.fromstring(''.join(f)) @@ -5376,7 +5608,7 @@ def streamfile(url, http_meth = http_GET, bufsize=8192, data=None, progress_obj= """ performs http_meth on url and read bufsize bytes from the response until EOF is reached. After each read bufsize bytes are yielded to the - caller. + caller. A spezial usage is bufsize="line" to read line by line (text). """ cl = '' retries = 0 @@ -5401,14 +5633,23 @@ def streamfile(url, http_meth = http_GET, bufsize=8192, data=None, progress_obj= if progress_obj: basename = os.path.basename(urlsplit(url)[2]) progress_obj.start(basename=basename, text=text, size=cl) - data = f.read(bufsize) - read = len(data) - while len(data): + + if bufsize == "line": + bufsize = 8192 + xread = f.readline + else: + xread = f.read + + read = 0 + while True: + data = xread(bufsize) + if not len(data): + break + read += len(data) if progress_obj: progress_obj.update(read) yield data - data = f.read(bufsize) - read += len(data) + if progress_obj: progress_obj.end(read) f.close() @@ -5419,7 +5660,7 @@ def streamfile(url, http_meth = http_GET, bufsize=8192, data=None, progress_obj= def buildlog_strip_time(data): """Strips the leading build time from the log""" - time_regex = re.compile('^\[\s{0,5}\d+s\]\s', re.M) + time_regex = re.compile('^\[[^\]]*\] ', re.M) return time_regex.sub('', data) @@ -5438,7 +5679,7 @@ def print_buildlog(apiurl, prj, package, repository, arch, offset=0, strip_time= query['start'] = offset start_offset = offset u = makeurl(apiurl, ['build', prj, repository, arch, package, '_log'], query=query) - for data in streamfile(u): + for data in streamfile(u, bufsize="line"): offset += len(data) if strip_time: data = buildlog_strip_time(data) @@ -5490,7 +5731,7 @@ def get_source_rev(apiurl, project, package, revision=None): # CAUTION: We have to loop through all rev and find the highest one, if none given. if revision: - url = makeurl(apiurl, ['source', project, package, '_history'], {'rev':revision}) + url = makeurl(apiurl, ['source', project, package, '_history'], {'rev': revision}) else: url = makeurl(apiurl, ['source', project, package, '_history']) f = http_GET(url) @@ -5503,7 +5744,7 @@ def get_source_rev(apiurl, project, package, revision=None): elif ent.find('time').text < new.find('time').text: ent = new if not ent: - return { 'version': None, 'error':'empty revisionlist: no such package?' } + return { 'version': None, 'error': 'empty revisionlist: no such package?' } e = {} for k in ent.keys(): e[k] = ent.get(k) @@ -5519,20 +5760,21 @@ def get_buildhistory(apiurl, prj, package, repository, arch, format = 'text'): r = [] for node in root.findall('entry'): - rev = int(node.get('rev')) + rev = node.get('rev') srcmd5 = node.get('srcmd5') versrel = node.get('versrel') bcnt = int(node.get('bcnt')) - t = time.localtime(int(node.get('time'))) - t = time.strftime('%Y-%m-%d %H:%M:%S', t) + t = time.gmtime(int(node.get('time'))) + t = time.strftime('%Y-%m-%d %H:%M:%S %Z', t) if format == 'csv': - r.append('%s|%s|%d|%s.%d' % (t, srcmd5, rev, versrel, bcnt)) + r.append('%s|%s|%s|%s.%d' % (t, srcmd5, rev, versrel, bcnt)) else: - r.append('%s %s %6d %s.%d' % (t, srcmd5, rev, versrel, bcnt)) + bversrel='%s.%d' % (versrel, bcnt) + r.append('%s %s %s %s' % (t, srcmd5, bversrel.ljust(16)[:16], rev)) if format == 'text': - r.insert(0, 'time srcmd5 rev vers-rel.bcnt') + r.insert(0, 'time srcmd5 vers-rel.bcnt rev') return r @@ -5557,11 +5799,11 @@ def print_jobhistory(apiurl, prj, current_package, repository, arch, format = 't reason = "unknown" code = node.get('code') rt = int(node.get('readytime')) - readyt = time.localtime(rt) - readyt = time.strftime('%Y-%m-%d %H:%M:%S', readyt) + readyt = time.gmtime(rt) + readyt = time.strftime('%Y-%m-%d %H:%M:%S %Z', readyt) st = int(node.get('starttime')) et = int(node.get('endtime')) - endtime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(et)) + endtime = time.strftime('%Y-%m-%d %H:%M:%S %Z', time.gmtime(et)) waittm = time.gmtime(et-st) if waittm.tm_mday > 1: waitbuild = "%1dd %2dh %2dm %2ds" % (waittm.tm_mday-1, waittm.tm_hour, waittm.tm_min, waittm.tm_sec) @@ -5622,8 +5864,8 @@ def get_commitlog(apiurl, prj, package, revision, format = 'text', meta = False, requestid = node.find('requestid').text.encode(locale.getpreferredencoding(), 'replace') except: requestid = "" - t = time.localtime(int(node.find('time').text)) - t = time.strftime('%Y-%m-%d %H:%M:%S', t) + t = time.gmtime(int(node.find('time').text)) + t = time.strftime('%Y-%m-%d %H:%M:%S %Z', t) if format == 'csv': s = '%s|%s|%s|%s|%s|%s|%s' % (rev, user, t, srcmd5, version, @@ -6196,7 +6438,7 @@ def setBugowner(apiurl, prj, pac, user=None, group=None): template_args=None, create_new=False) if user.startswith('group:'): - group=user.replace('group:','') + group=user.replace('group:', '') user=None if data: root = ET.fromstring(''.join(data)) @@ -6286,6 +6528,7 @@ def addGitSource(url): si = Serviceinfo() s = si.addGitUrl(services, url) s = si.addRecompressTar(services) + s = si.addSetVersion(services) si.read(s) # for pretty output @@ -6850,7 +7093,7 @@ def find_default_project(apiurl=None, package=None): # any fast query will do here. show_package_meta(apiurl, prj, package) return prj - except HTTPError: + except HTTPError: pass return None @@ -6865,4 +7108,17 @@ def utime(filename, arg, ignore_einval=True): return raise +def which(name): + """Searches "name" in PATH.""" + name = os.path.expanduser(name) + if os.path.isabs(name): + if os.path.exists(name): + return name + return None + for directory in os.environ.get('PATH', '').split(':'): + path = os.path.join(directory, name) + if os.path.exists(path): + return path + return None + # vim: sw=4 et diff --git a/osc/fetch.py b/osc/fetch.py index fb2248b..95643e1 100644 --- a/osc/fetch.py +++ b/osc/fetch.py @@ -239,6 +239,12 @@ def run(self, buildinfo): i.makeurls(self.cachedir, self.urllist) if os.path.exists(i.fullfilename): cached += 1 + if i.hdrmd5: + from .util import packagequery + hdrmd5 = packagequery.PackageQuery.queryhdrmd5(i.fullfilename) + if not hdrmd5 or hdrmd5 != i.hdrmd5: + os.unlink(i.fullfilename) + cached -= 1 miss = 0 needed = all - cached if all: @@ -254,6 +260,10 @@ def run(self, buildinfo): '--offline not possible.' % i.fullfilename) self.dirSetup(i) + if i.hdrmd5 and self.enable_cpio: + self.__add_cpio(i) + done += 1 + continue try: # if there isn't a progress bar, there is no output at all if not self.progress_obj: diff --git a/osc/oscssl.py b/osc/oscssl.py index 72a0dde..325e1ab 100644 --- a/osc/oscssl.py +++ b/osc/oscssl.py @@ -271,7 +271,7 @@ def _start_ssl(self): def endheaders(self, *args, **kwargs): if self._proxy_auth is None: self._proxy_auth = self._encode_auth() - HTTPSConnection.endheaders(self, *args, **kwargs) + HTTPSConnection.endheaders(self, *args, **kwargs) # broken in m2crypto: port needs to be an int def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0): diff --git a/osc/util/archquery.py b/osc/util/archquery.py index fdafefc..c39c72a 100644 --- a/osc/util/archquery.py +++ b/osc/util/archquery.py @@ -88,6 +88,12 @@ def provides(self): def requires(self): return self.fields['depend'] if 'depend' in self.fields else [] + def conflicts(self): + return self.fields['conflict'] if 'conflict' in self.fields else [] + + def obsoletes(self): + return self.fields['replaces'] if 'replaces' in self.fields else [] + def canonname(self): pkgver = self.fields['pkgver'][0] if 'pkgver' in self.fields else None return self.name() + '-' + pkgver + '-' + self.arch() + '.' + self.pkgsuffix diff --git a/osc/util/cpio.py b/osc/util/cpio.py index 44351ea..cd3c406 100644 --- a/osc/util/cpio.py +++ b/osc/util/cpio.py @@ -64,7 +64,7 @@ def __init__(self, mgc, ino, mode, uid, gid, nlink, mtime, filesize, self.namesize = namesize # != 0 indicates CRC format (which we do not support atm) self.checksum = checksum - for k,v in self.__dict__.items(): + for k, v in self.__dict__.items(): self.__dict__[k] = int(v, 16) self.filename = filename # data starts at dataoff and ends at dataoff+filesize @@ -82,7 +82,7 @@ class CpioRead: # supported formats - use name -> mgc mapping to increase readabilty sfmt = { - 'newascii' : '070701', + 'newascii': '070701', } # header format diff --git a/osc/util/debquery.py b/osc/util/debquery.py index aaf6f22..220fd2a 100644 --- a/osc/util/debquery.py +++ b/osc/util/debquery.py @@ -13,7 +13,7 @@ class DebError(packagequery.PackageError): class DebQuery(packagequery.PackageQuery): default_tags = ('package', 'version', 'release', 'epoch', 'architecture', 'description', - 'provides', 'depends', 'pre_depends') + 'provides', 'depends', 'pre_depends', 'conflicts', 'breaks') def __init__(self, fh): self.__file = fh @@ -71,9 +71,11 @@ def __parse_control(self, control, all_tags=False, self_provides=True, *extra_ta self.fields['provides'] = [ i.strip() for i in re.split(',\s*', self.fields.get('provides', '')) if i ] self.fields['depends'] = [ i.strip() for i in re.split(',\s*', self.fields.get('depends', '')) if i ] self.fields['pre_depends'] = [ i.strip() for i in re.split(',\s*', self.fields.get('pre_depends', '')) if i ] + self.fields['conflicts'] = [ i.strip() for i in re.split(',\s*', self.fields.get('conflicts', '')) if i ] + self.fields['breaks'] = [ i.strip() for i in re.split(',\s*', self.fields.get('breaks', '')) if i ] if self_provides: # add self provides entry - self.fields['provides'].append('%s = %s' % (self.name(), '-'.join(versrel))) + self.fields['provides'].append('%s (= %s)' % (self.name(), '-'.join(versrel))) def vercmp(self, debq): res = cmp(int(self.epoch()), int(debq.epoch())) @@ -110,7 +112,13 @@ def provides(self): return self.fields['provides'] def requires(self): - return self.fields['depends'] + return self.fields['depends'] + self.fields['pre_depends'] + + def conflicts(self): + return self.fields['conflicts'] + self.fields['breaks'] + + def obsoletes(self): + return [] def gettag(self, num): return self.fields.get(num, None) diff --git a/osc/util/packagequery.py b/osc/util/packagequery.py index a1f373f..86636f7 100644 --- a/osc/util/packagequery.py +++ b/osc/util/packagequery.py @@ -15,7 +15,7 @@ class PackageQueries(dict): """ # map debian arches to common obs arches - architectureMap = {'i386': ['i586', 'i686'], 'amd64': ['x86_64']} + architectureMap = {'i386': ['i586', 'i686'], 'amd64': ['x86_64'], 'ppc64el': ['ppc64le']} # map rpm arches to mer obs scheduler arches architectureMap.update({'armv7hl': ['armv8el'], 'armv7tnhl': ['armv8el'], @@ -81,6 +81,12 @@ def provides(self): def requires(self): raise NotImplementedError + def conflicts(self): + raise NotImplementedError + + def obsoletes(self): + raise NotImplementedError + def gettag(self): raise NotImplementedError @@ -90,6 +96,13 @@ def vercmp(self, pkgquery): def canonname(self): raise NotImplementedError + def evr(self): + evr = self.version() + "-" + self.release() + epoch = self.epoch() + if epoch is not None and epoch != 0: + evr = epoch + ":" + evr + return evr + @staticmethod def query(filename, all_tags=False, extra_rpmtags=(), extra_debtags=(), self_provides=True): f = open(filename, 'rb') @@ -117,6 +130,17 @@ def query(filename, all_tags=False, extra_rpmtags=(), extra_debtags=(), self_pro f.close() return pkgquery + @staticmethod + def queryhdrmd5(filename): + f = open(filename, 'rb') + magic = f.read(7) + f.seek(0) + if magic[:4] == '\xed\xab\xee\xdb': + from . import rpmquery + f.close() + return rpmquery.RpmQuery.queryhdrmd5(filename) + return None + if __name__ == '__main__': import sys try: diff --git a/osc/util/rpmquery.py b/osc/util/rpmquery.py index bb85028..2ebc677 100644 --- a/osc/util/rpmquery.py +++ b/osc/util/rpmquery.py @@ -60,7 +60,9 @@ class RpmQuery(packagequery.PackageQuery): default_tags = (1000, 1001, 1002, 1003, 1004, 1022, 1005, 1020, 1047, 1112, 1113, # provides - 1049, 1048, 1050 # requires + 1049, 1048, 1050, # requires + 1054, 1053, 1055, # conflicts + 1090, 1114, 1115 # obsoletes ) def __init__(self, fh): @@ -69,7 +71,7 @@ def __init__(self, fh): self.filename_suffix = 'rpm' self.header = None - def read(self, all_tags=False, self_provides=True, *extra_tags): + def read(self, all_tags=False, self_provides=True, *extra_tags, **extra_kw): # self_provides is unused because a rpm always has a self provides self.__read_lead() data = self.__file.read(RpmHeaderEntry.ENTRY_SIZE) @@ -80,8 +82,10 @@ def read(self, all_tags=False, self_provides=True, *extra_tags): size = il * RpmHeaderEntry.ENTRY_SIZE + dl # data is 8 byte aligned pad = (size + 7) & ~7 - self.__file.read(pad) - data = self.__file.read(RpmHeaderEntry.ENTRY_SIZE) + querysig = extra_kw.get('querysig') + if not querysig: + self.__file.read(pad) + data = self.__file.read(RpmHeaderEntry.ENTRY_SIZE) hdrmgc, reserved, il, dl = struct.unpack('!I3i', data) self.header = RpmHeader(pad, dl) if self.HEADER_MAGIC != hdrmgc: @@ -115,9 +119,10 @@ def __read_data(self, entry, data): entry.data = struct.unpack('!%dh' % entry.count, data[off:off + 2 * entry.count]) elif entry.type == 4: entry.data = struct.unpack('!%di' % entry.count, data[off:off + 4 * entry.count]) - elif entry.type == 6 or entry.type == 7: - # XXX: what to do with binary data? for now treat it as a string + elif entry.type == 6: entry.data = unpack_string(data[off:]) + elif entry.type == 7: + entry.data = data[off:off + entry.count] elif entry.type == 8 or entry.type == 9: cnt = entry.count entry.data = [] @@ -151,7 +156,10 @@ def __read_data(self, entry, data): raise RpmHeaderError(self.__path, 'unsupported tag type \'%d\' (tag: \'%s\'' % (entry.type, entry.tag)) def __reqprov(self, tag, flags, version): - pnames = self.header.gettag(tag).data + pnames = self.header.gettag(tag) + if not pnames: + return [] + pnames = pnames.data pflags = self.header.gettag(flags).data pvers = self.header.gettag(version).data if not (pnames and pflags and pvers): @@ -221,6 +229,12 @@ def provides(self): def requires(self): return self.__reqprov(1049, 1048, 1050) + def conflicts(self): + return self.__reqprov(1054, 1053, 1055) + + def obsoletes(self): + return self.__reqprov(1090, 1114, 1115) + def is_src(self): # SOURCERPM = 1044 return self.gettag(1044) is None @@ -250,6 +264,17 @@ def query(filename): f.close() return rpmq + @staticmethod + def queryhdrmd5(filename): + f = open(filename, 'rb') + rpmq = RpmQuery(f) + rpmq.read(1004, querysig=True) + f.close() + entry = rpmq.gettag(1004) + if entry is None: + return None + return ''.join([ "%02x" % x for x in struct.unpack('16B', entry.data) ]) + @staticmethod def rpmvercmp(ver1, ver2): """ @@ -326,3 +351,5 @@ def unpack_string(data): print('\n'.join(rpmq.provides())) print('##########') print('\n'.join(rpmq.requires())) + print('##########') + print(RpmQuery.queryhdrmd5(sys.argv[1])) diff --git a/rpm/osc.spec b/rpm/osc.spec new file mode 100644 index 0000000..2c4b9f2 --- /dev/null +++ b/rpm/osc.spec @@ -0,0 +1,91 @@ +# +# Do NOT Edit the Auto-generated Part! +# Generated by: spectacle version 0.27 +# + +Name: osc + +# >> macros +# << macros + +%{!?python_sitelib: %define python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} +Summary: OpenSUSE Build Service Commander +Version: 0.146.0 +Release: 1 +Group: Development/Tools/Other +License: GPL v2 or later +BuildArch: noarch +URL: http://www.gitorious.org/opensuse/osc +Source0: osc-%{version}.tar.bz2 +Source100: osc.yaml +Requires: python-urlgrabber +Requires: m2crypto > 0.19 +Requires: /usr/bin/less +Requires: /usr/bin/diff3 +BuildRequires: python-devel +BuildRequires: python-urlgrabber +BuildRequires: m2crypto + +%description +Commandline client for the openSUSE Build Service. + +See http://en.opensuse.org/Build_Service/CLI , as well as +http://en.opensuse.org/Build_Service_Tutorial for a general +introduction. + + +%prep +%setup -q + +# >> setup +# << setup + +%build +# >> build pre +# << build pre + +CFLAGS="$RPM_OPT_FLAGS" %{__python} setup.py build + +# >> build post +# << build post + +%install +rm -rf %{buildroot} +# >> install pre +# << install pre +%{__python} setup.py install --root=%{buildroot} -O1 + +# >> install post +ln -s osc-wrapper.py %{buildroot}/%{_bindir}/osc +mkdir -p %{buildroot}/var/lib/osc-plugins +mkdir -p %{buildroot}%{_sysconfdir}/profile.d +install -m 0755 dist/complete.csh %{buildroot}%{_sysconfdir}/profile.d/osc.csh +install -m 0755 dist/complete.sh %{buildroot}%{_sysconfdir}/profile.d/osc.sh +%if 0%{?suse_version} > 1110 +mkdir -p %{buildroot}%{_prefix}/lib/osc +install -m 0755 dist/osc.complete %{buildroot}%{_prefix}/lib/osc/complete +%else +mkdir -p %{buildroot}%{_prefix}/%{_lib}/osc +install -m 0755 dist/osc.complete %{buildroot}%{_prefix}/%{_lib}/osc/complete +%endif + + +# << install post + +%files +%defattr(-,root,root,-) +# >> files +%{_bindir}/osc* +%{python_sitelib}/* +%{_sysconfdir}/profile.d/* +%if 0%{?suse_version} > 1110 +%dir %{_prefix}/lib/osc +%{_prefix}/lib/osc/* +%else +%dir %{_prefix}/%{_lib}/osc +%{_prefix}/%{_lib}/osc/* +%endif +%dir /var/lib/osc-plugins +%doc AUTHORS README TODO NEWS +%doc %_mandir/man1/osc.* +# << files diff --git a/rpm/osc.yaml b/rpm/osc.yaml new file mode 100644 index 0000000..6d29eb3 --- /dev/null +++ b/rpm/osc.yaml @@ -0,0 +1,30 @@ +Name: osc +Summary: OpenSUSE Build Service Commander +Version: 0.146.0 +Release: 1 +Group: Development/Tools/Other +License: GPL v2 or later +URL: http://www.gitorious.org/opensuse/osc +Sources: + - osc-%{version}.tar.bz2 +Description: | + Commandline client for the openSUSE Build Service. + + See http://en.opensuse.org/Build_Service/CLI , as well as + http://en.opensuse.org/Build_Service_Tutorial for a general + introduction. +Requires: + - python-urlgrabber + - m2crypto > 0.19 + - /usr/bin/less + - /usr/bin/diff3 +PkgBR: + - python-devel + - python-urlgrabber + - m2crypto +Configure: none +Builder: python +BuildArch: noarch +Recommends: + - build > 2010.11.24 +SetupOptions: -q diff --git a/tests/request_fixtures/test_read_request1.xml b/tests/request_fixtures/test_read_request1.xml index 8780781..af69950 100644 --- a/tests/request_fixtures/test_read_request1.xml +++ b/tests/request_fixtures/test_read_request1.xml @@ -7,7 +7,8 @@ - + + Create Request foobar title of the request diff --git a/tests/request_fixtures/test_read_request2.xml b/tests/request_fixtures/test_read_request2.xml index 52d5808..2916955 100644 --- a/tests/request_fixtures/test_read_request2.xml +++ b/tests/request_fixtures/test_read_request2.xml @@ -15,5 +15,7 @@ review start - + + Created request + diff --git a/tests/request_fixtures/test_request_list_view2.xml b/tests/request_fixtures/test_request_list_view2.xml index 976c155..ae9213a 100644 --- a/tests/request_fixtures/test_request_list_view2.xml +++ b/tests/request_fixtures/test_request_list_view2.xml @@ -4,8 +4,12 @@ - - + + Created Request + + + Review Approved + This is a simple request with a lot of ... ... text and other stuff. This request also contains a description. This is useful to diff --git a/tests/test_commit.py b/tests/test_commit.py index 495e9c8..aed14b4 100644 --- a/tests/test_commit.py +++ b/tests/test_commit.py @@ -295,7 +295,7 @@ def test_added_missing2(self): file='testSimple_missingfilelist', expfile='testSimple_lfilelist') @PUT('http://localhost/source/osctest/simple/nochange?rev=repository', exp='This file didn\'t change but\nis modified.\n', text=rev_dummy) - @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin', + @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin', expfile='testSimple_lfilelist', text='an error occured', code=500) def test_commitfilelist_error(self): """commit modified file but when committing the filelist the server returns status 500 (see ticket #65)""" diff --git a/tests/test_request.py b/tests/test_request.py index 5b941c0..4fe2fb7 100644 --- a/tests/test_request.py +++ b/tests/test_request.py @@ -342,7 +342,6 @@ def test_read_request1(self): self.assertEqual(r.state.when, '2010-12-27T01:36:29') self.assertEqual(r.state.who, 'user1') self.assertEqual(r.state.comment, '') - self.assertEqual(r.statehistory[0].name, 'new') self.assertEqual(r.statehistory[0].when, '2010-12-13T13:02:03') self.assertEqual(r.statehistory[0].who, 'creator') self.assertEqual(r.statehistory[0].comment, 'foobar') @@ -382,7 +381,6 @@ def test_read_request2(self): self.assertEqual(r.reviews[0].who, 'abc') self.assertEqual(r.reviews[0].comment, 'review start') self.assertTrue(r.reviews[0].by_user is None) - self.assertEqual(r.statehistory[0].name, 'new') self.assertEqual(r.statehistory[0].when, '2010-12-11T00:00:00') self.assertEqual(r.statehistory[0].who, 'creator') self.assertEqual(r.statehistory[0].comment, '') @@ -455,7 +453,7 @@ def test_request_list_view2(self): exp = """\ 21 State:accepted By:foobar When:2010-12-29T16:37:45 set_bugowner: buguser foo - From: new(user) -> review(foobar) + From: Created Request: user -> Review Approved: foobar Descr: This is a simple request with a lot of ... ... text and other stuff. This request also contains a description. This is useful to describe the request. blabla blabla\n""" @@ -472,7 +470,7 @@ def test_request_str1(self): exp = """\ Request: #123 - submit: xyz/abc(cleanup) -> foo + submit: xyz/abc(cleanup) -> foo ***update link*** add_role: person: bar as maintainer, group: groupxyz as reader home:foo @@ -488,8 +486,8 @@ def test_request_str1(self): Review: accepted Group: group1 2010-12-29T00:11:22 abc accepted new Group: group1 2010-12-28T00:11:22 abc review start -History: revoked 2010-12-12T00:00:00 creator - new 2010-12-11T00:00:00 creator""" +History: 2010-12-12T00:00:00 creator revoked + 2010-12-11T00:00:00 creator new""" self.assertEqual(exp, str(r)) def test_request_str2(self): diff --git a/tests/test_update.py b/tests/test_update.py index 27c1c62..aaffcab 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -70,7 +70,7 @@ def testUpdateUpstreamModifiedFile(self): @GET('http://localhost/source/osctest/conflict/_meta', file='meta.xml') def testUpdateConflict(self): """ - a file was modified in the remote package (local file is also modified + a file was modified in the remote package (local file is also modified and a merge isn't possible) """ self._change_to_pkg('conflict')