#!/usr/bin/env tclsh # # MREL, build release files and test them # Copyright (C) 2020-2022 Xavier Delaruelle # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . ########################################################################## proc reportUsage {} { puts "Usage: $::argv0 \[options\] Build Modules release files and test them Options: -h, --help Show this help message and exit" } proc sgr {sgrcode str} { return "\033\[${sgrcode}m$str\033\[0m" } proc logadd {str} { if {[info exists ::logfid]} { puts $::logfid $str } } proc reportInfo {str} { logadd "--- $str" puts [sgr 2 $str] } proc reportError {str} { logadd "### ERROR: $str" puts "[sgr {1;31} ERROR]: $str" } proc runcmd {args} { reportInfo "Running command: $args" eval exec >@$::logfid $args } proc ignoreexp {errmsg expmsg} { if {[string first $expmsg $errmsg] == -1} { error $errmsg } } proc quitorcont {} { flush stdout puts -nonewline "Is this ok \[Y/n\]: " flush stdout gets stdin ok if {$ok ne {} && ![string equal -nocase -length 1 {y} $ok]} { error Abort } } proc extractflist {relver distcontentfile listfile} { set fid [open $distcontentfile r] set distfiles [read $fid] close $fid set fid [open $listfile w] puts $fid [join [lsort [split [string map [list modules-$relver/ {}]\ $distfiles] \n]] \n] close $fid } # parse arguments set hintmsg "\n Try '$argv0 --help' for more information." if {$argc > 1} { reportError "Unexpected number of arguments$hintmsg" exit 1 } elseif {$argc == 1} { switch -- [lindex $argv 0] { -h - --help { reportUsage exit 0 } default { reportError "Invalid option '[lindex $argv 0]'$hintmsg" exit 1 } } } # surround whole code to catch error and quit properly if {[catch { set exitcode 0 set rpmbuilddir $env(HOME)/rpmbuild set srpmdir $rpmbuilddir/SRPMS set rpmdir $rpmbuilddir/RPMS/$tcl_platform(machine) set dlurl https://github.com/cea-hpc/modules/releases/download set cwd [pwd] # define and open log file set logfile mrel.out set logfid [open $logfile w] # get current branch set relbranch [exec git branch --show-current] if {[regexp {^(main|v\d+.\d+.x)$} $relbranch]} { reportInfo "Found branch '$relbranch'" } else { error "git branch '$relbranch' is not a valid release branch" } # ensure head sit on a tag set reltag [exec git describe --tags --exact-match] reportInfo "Found release tag '$reltag'" if {[regexp {^v(\d+.\d+.\d+(-(alpha|beta))?)$} $reltag match relver]} { reportInfo "Extract release version number '$relver'" } else { error "git tag '$reltag' is not a valid release tag" } # get previous version for later comparison set prevtag [exec git describe --tags --abbrev=0 HEAD^] reportInfo "Found previous release tag '$prevtag'" if {[regexp {^v(\d+.\d+.\d+(-(alpha|beta))?)$} $prevtag match prevver]} { reportInfo "Extract previous release version number '$prevver'" } else { error "git tag '$prevtag' is not a valid release tag" } # acquire credential required to locally install reportInfo "Running command: sudo --validate" exec sudo --validate reportInfo {sudo credential acquired} # acquire credential to build on remote fedora systems set fedora_realm FEDORAPROJECT.ORG if {![info exists env(MREL_FEDORA_USERNAME)]} { puts -nonewline "$fedora_realm username: " flush stdout gets stdin env(MREL_FEDORA_USERNAME) } set fedora_princ $env(MREL_FEDORA_USERNAME)@$fedora_realm reportInfo "Fedora principal set to '$fedora_princ'" if {[catch {exec klist} klist_out] || ![string match "*Default principal:\ $fedora_princ*" $klist_out]} { reportInfo "Running command: kinit $fedora_princ" exec >@stdout kinit $fedora_princ } reportInfo "Kerberos ticket to $fedora_realm realm acquired" # get name of GitHub remote repository used to trigger CI if {![info exists env(MREL_GITHUB_TEST_REMOTE)]} { puts -nonewline "GitHub test remote: " flush stdout gets stdin env(MREL_GITHUB_TEST_REMOTE) } set gh_test_remote $env(MREL_GITHUB_TEST_REMOTE) reportInfo "GitHub test remote set to '$gh_test_remote'" set ghremtgt [lindex [split [exec git remote get-url $gh_test_remote] :.] 2] set ghstatusurl\ https://api.github.com/repos/$ghremtgt/commits/$relbranch/status set ghtgturl https://github.com/$ghremtgt/commit/$relbranch set ghcloneurl https://github.com/$ghremtgt.git set ghexporturl https://github.com/$ghremtgt/archive/$reltag.tar.gz # clean previous release files file delete -force /tmp/mods+test-gz /tmp/mods+test-bz2 /tmp/mods+test-clone\ /tmp/mods+test-export file mkdir /tmp/mods+test-gz /tmp/mods+test-bz2 /tmp/mods+test-clone\ /tmp/mods+test-export reportInfo {Recreate test directory /tmp/mods+test-gz and /tmp/mods+test-bz2\ /tmp/mods+test-clone /tmp/mods+test-export} foreach oldsrpm [glob -nocomplain $srpmdir/environment-modules-*.src.rpm] { reportInfo "Delete previous SRPM '$oldsrpm'" file delete $oldsrpm } foreach oldrpm [glob -nocomplain $rpmdir/environment-modules-*.rpm] { reportInfo "Delete previous RPM '$oldrpm'" file delete $oldrpm } # remove installed RPMs or dist catch {runcmd 2>@$logfid sudo rpm -e --nodeps scl-utils} catch {runcmd 2>@$logfid sudo rpm -e --nodeps environment-modules} catch {runcmd 2>@$logfid sudo rpm -e --nodeps environment-modules-compat} runcmd sudo rm -rf /usr/local/Modules # Phase 1: build dists and verify them # --------------------------------------------------------- if {[catch {runcmd make distclean} errmsg]} { # skip error if things have already been cleaned up ignoreexp $errmsg {*** Makefile.inc is missing, please run\ './configure'. Stop.} } runcmd ./configure # download icdiff to compare files runcmd >&@$logfid make icdiff # build all dists runcmd make dist runcmd make dist-bzip2 runcmd make dist-win runcmd make srpm set distgz modules-$relver.tar.gz set distbz modules-$relver.tar.bz2 set distwin modules-$relver-win.zip set distgzsum [lindex [exec md5sum $distgz] 0] set distbzsum [lindex [exec md5sum $distbz] 0] set distwinsum [lindex [exec md5sum $distwin] 0] reportInfo "Found dist GZ '$distgz' (size='[file size $distgz]',\ sum='$distgzsum')" reportInfo "Found dist BZ '$distbz' (size='[file size $distbz]',\ sum='$distbzsum')" reportInfo "Found dist Win '$distwin' (size='[file size $distwin]',\ sum='$distwinsum')" # adapt rpm release version in case of non-final release (alpha/beta) if {[set nonfinalidx [string first - $relver]] != -1} { set rpmrelver [string range $relver 0 [expr {$nonfinalidx -\ 1}]]-0.1.[string range $relver [expr {$nonfinalidx + 1}] end] } else { set rpmrelver $relver-1 } set srcrpm [glob $srpmdir/environment-modules-$rpmrelver.*.src.rpm] reportInfo "Found source RPM '$srcrpm' (size='[file size $srcrpm]')" quitorcont runcmd >log-gz tar tfz $distgz runcmd >log-bz tar tfj $distbz runcmd diff -u log-gz log-bz runcmd >log-win zipinfo -1 $distwin # prepare current version dist file list to compare to previous dist extractflist $relver log-gz distfiles extractflist $relver-win log-win distfiles-win # fetch previous dists set prevdistgz modules-$prevver.tar.gz set prevdistwin modules-$prevver-win.zip runcmd 2>@$logfid wget -O $prevdistgz $dlurl/$prevtag/$prevdistgz runcmd 2>@$logfid wget -O $prevdistwin $dlurl/$prevtag/$prevdistwin # prepare previous version dist file list to compare to previous dist runcmd >log-prev-gz tar tfz $prevdistgz extractflist $prevver log-prev-gz distfiles-prev runcmd >log-prev-win zipinfo -1 $prevdistwin extractflist $prevver-win log-prev-win distfiles-win-prev # compare dist content with previous release runcmd >@stdout ./icdiff distfiles-prev distfiles | less -eFKRX quitorcont runcmd >@stdout ./icdiff distfiles-win-prev distfiles-win | less -eFKRX quitorcont file delete log-gz log-bz log-win $prevdistgz distfiles distfiles-win\ log-prev-gz distfiles-prev $prevdistwin log-prev-win distfiles-win-prev # check generated RPM spec file runcmd >@stdout rpmlint -r contrib/rpm/environment-modules.rpmlintrc\ contrib/rpm/environment-modules.spec quitorcont # Phase 2: push to CI # --------------------------------------------------------- runcmd 2>@$logfid git push --force $gh_test_remote c-3.2 runcmd 2>@$logfid git push --force $gh_test_remote $relbranch runcmd 2>@$logfid git push --force $gh_test_remote $reltag # see Phase 7 for CI result check # Phase 3: build, test, install from git repository # --------------------------------------------------------- runcmd make runcmd script/mt runcmd 2>@$logfid sudo make install runcmd script/mt install runcmd 2>@$logfid sudo make uninstall # Phase 4: build, test, install from generated dists # --------------------------------------------------------- cd /tmp/mods+test-gz runcmd tar xfz $cwd/$distgz cd modules-$relver reportInfo "Moved into '[pwd]' directory" runcmd ./configure runcmd make runcmd 2>@$logfid script/mt runcmd 2>@$logfid sudo make install runcmd script/mt install runcmd 2>@$logfid sudo make uninstall runcmd make clean cd /tmp/mods+test-bz2 runcmd tar xfj $cwd/$distbz cd modules-$relver reportInfo "Moved into '[pwd]' directory" runcmd ./configure runcmd make runcmd 2>@$logfid script/mt runcmd 2>@$logfid sudo make install runcmd script/mt install runcmd 2>@$logfid sudo make uninstall runcmd make clean cd /tmp/mods+test-clone runcmd 2>@$logfid git clone $ghcloneurl cd modules reportInfo "Moved into '[pwd]' directory" runcmd 2>@$logfid git checkout $relbranch runcmd 2>@$logfid ./configure runcmd make runcmd 2>@$logfid script/mt quick runcmd 2>@$logfid sudo make install runcmd script/mt install runcmd 2>@$logfid sudo make uninstall runcmd make clean cd /tmp/mods+test-export runcmd 2>@$logfid wget -O $reltag.tar.gz $ghexporturl runcmd tar xfz $reltag.tar.gz cd modules-$relver reportInfo "Moved into '[pwd]' directory" runcmd ./configure runcmd make runcmd 2>@$logfid script/mt quick runcmd 2>@$logfid sudo make install runcmd script/mt install runcmd 2>@$logfid sudo make uninstall runcmd make clean cd $cwd reportInfo "Moved back into '[pwd]' directory" # Phase 5: build, test, install from RPM built locally # --------------------------------------------------------- runcmd 2>@$logfid rpmbuild --rebuild $srcrpm foreach rpmpkg [glob $rpmdir/environment-modules-$rpmrelver.*.rpm] { runcmd >@stdout rpm -qlp $rpmpkg | less -eFKRX quitorcont # check installation of built RPM runcmd sudo rpm -ivh $rpmpkg } # Phase 6: build and test on Fedora platform # --------------------------------------------------------- # get list of targets set koji_out [exec koji list-targets --quiet] foreach {match ftarget} [regexp -all -inline -line {^(f\d+|epel\d+) }\ $koji_out] { lappend fedora_target_list $ftarget } reportInfo "Use Fedora targets: $fedora_target_list" # submit builds foreach ftarget $fedora_target_list { set koji_out [exec koji build --scratch $ftarget $srcrpm --nowait\ --noprogress] if {[regexp {Created task: (\d+)} $koji_out match taskid]} { reportInfo "Submitted task '$taskid' on '$ftarget' target" } else { error "cannot find koji task submitted to $ftarget" } set koji_tasks($taskid) $ftarget } # monitor build completeness set koji_taskinfo_url https://koji.fedoraproject.org/koji/taskinfo?taskID= while {[array size koji_tasks] > 0} { foreach ktask [array names koji_tasks] { if {[catch { set koji_out [exec koji taskinfo $ktask] regexp {State: (\w+)} $koji_out match kstate switch -- $kstate { free - open { reportInfo "Task '$ktask' on '$koji_tasks($ktask)' still\ running" } closed { reportInfo "Task '$ktask' on '$koji_tasks($ktask)'\ successfully completed" unset koji_tasks($ktask) } default { reportError "Task '$ktask' on '$koji_tasks($ktask)' failed,\ see $koji_taskinfo_url$ktask" unset koji_tasks($ktask) set exitcode 1 } } } errmsg]} { # allow for 1 failure if {[info exists koji_fail($ktask)]} { error $errmsg } else { reportInfo "Cannot fetch state of task '$ktask' on\ '$koji_tasks($ktask)', will be retried" set koji_fail($ktask) 1 } # clear previous failure if it succeed this time } elseif {[info exists koji_fail($ktask)]} { unset koji_fail($ktask) } } # wait 30 seconds before next check loop if {[array size koji_tasks] > 0} { after 30000 } } # Phase 7: verify CI result # --------------------------------------------------------- reportInfo "Checking CI tests status on $ghstatusurl" while {[set cistatus [lindex [split [string range [exec wget -q -O -\ $ghstatusurl] 0 30] \"] 3]] eq {pending}} { reportInfo "CI tests still in progress" # wait 30 seconds before next check loop after 30000 } if {$cistatus eq {success}} { reportInfo "CI tests are successful, see $ghtgturl" } else { reportError "CI tests have failed, see $ghtgturl" set exitcode 1 } # green light for mpub exec >mpub.ok md5sum $distgz $distbz $distwin # exit in error if any occurred } errmsg]} { reportError $errmsg set exitcode 1 } close $logfid exit $exitcode # vim:set tabstop=3 shiftwidth=3 expandtab autoindent syntax=tcl: