[PC-BSD Commits] r19647 - in pcbsd/current/src-sh/pbi-manager: . man8

svn at pcbsd.org svn at pcbsd.org
Thu Oct 11 10:36:36 PDT 2012


Author: kris
Date: 2012-10-11 17:36:35 +0000 (Thu, 11 Oct 2012)
New Revision: 19647

Modified:
   pcbsd/current/src-sh/pbi-manager/man8/pbi_autobuild.8
   pcbsd/current/src-sh/pbi-manager/pbi-manager
Log:

Large update to pbi_autobuild

Add new '-p <num>' option to set the number of concurrent builds you want
to run. On systems with multiple CPUs and Disk IO to spare this can greatly
speed up the build process. 

Also change the way we build the initial queue, so it only builds a single
time initially, then ever 2 hours after that to check for updated modules. 
This allows us to spend less time when spinning up builds. 



Modified: pcbsd/current/src-sh/pbi-manager/man8/pbi_autobuild.8
===================================================================
--- pcbsd/current/src-sh/pbi-manager/man8/pbi_autobuild.8	2012-10-11 16:34:25 UTC (rev 19646)
+++ pcbsd/current/src-sh/pbi-manager/man8/pbi_autobuild.8	2012-10-11 17:36:35 UTC (rev 19647)
@@ -8,8 +8,9 @@
 .Nm
 .Op Fl c Ar confdir
 .Op Fl d Ar portdir
+.Op Fl h Ar script
 .Op Fl o Ar outdir
-.Op Fl h Ar script
+.Op Fl p Ar <num>
 .Op Fl 32
 .Op Fl -genpatch
 .Op Fl -keep Ar num
@@ -41,6 +42,10 @@
 .It Fl o Ar outdir
 The directory to place the finished PBI files, will also be used to determine
 which apps are in need of a rebuild if the associated FreeBSD port has been updated.
+.It Fl p Ar <number>
+Number of build processes to run concurrently. Defaults to a single process, but if you
+have CPUs and disk IO to spare, increasing will greatly reduce time to do an i
+autobuild run.
 .It Fl 32
 Use to specify to build i386 PBIs on an amd64 host system
 .It Fl -genpatch

Modified: pcbsd/current/src-sh/pbi-manager/pbi-manager
===================================================================
--- pcbsd/current/src-sh/pbi-manager/pbi-manager	2012-10-11 16:34:25 UTC (rev 19646)
+++ pcbsd/current/src-sh/pbi-manager/pbi-manager	2012-10-11 17:36:35 UTC (rev 19647)
@@ -60,6 +60,7 @@
   -d portdir   -- Use different ports dir (Default: /usr/ports)
   -h script    -- Call the following helper script after each build
   -o outdir    -- Where to place the finished PBI file(s) <required>
+  -p <num>     -- Number of PBI builds to run concurrently (Defaults to 1)
   -32          -- Build i386 PBIs on amd64 host
   --genpatch   -- Generate patch files (*.pbp) from archived PBIs to current
   --keep <num> -- Keep <num> old versions in archive folder for each built PBI
@@ -986,6 +987,13 @@
                         get_abspath "$1"
                         PBI_AB_OUTDIR="$_ABSPATH"
                         ;;
+                -p)	if [ $# -eq 1 ]; then usage_autob_pbi; fi
+                        shift
+                        PBI_AB_BUILDERS="$1"
+			if [ ! $(is_num "$PBI_AB_BUILDERS") ] ; then 
+				exit_err "Invalid process number specifed!"
+			fi
+                        ;;
                 -h)	if [ $# -eq 1 ]; then usage_autob_pbi; fi
                         shift
                         get_abspath "$1"
@@ -1207,6 +1215,7 @@
         if [ -z "${PBI_MAKEPORT}" ]; then
                 usage_make_pbi
         fi
+
 }
 
 # Parse the update command line
@@ -1315,6 +1324,7 @@
 	else
 		PBI_HASHDIR="${PBI_APPDIR}/.hashdir-`whoami`"
 	fi
+	PBI_AB_BUILDERS=1
 	PBI_XDGCFGDIR="${PBI_APPDIR}/etc/xdg/menus"
 	PBI_XDGAPPDIR="${PBI_APPDIR}/share/applications"
 	PBI_XDGDIRDIR="${PBI_APPDIR}/share/desktop-directories"
@@ -1468,6 +1478,11 @@
 	PBI_VERBOSE="NO"
 	PORTSDIR="/usr/ports"
 
+}
+
+detect_pkgng()
+{
+	export PORTSDIR
 	# Are we using pkgng?
 	WITH_PKGNG=$(make -f ${PORTSDIR}/Mk/bsd.port.mk -V WITH_PKGNG)
         if [ -n "${WITH_PKGNG}" ]; then
@@ -5537,6 +5552,31 @@
         do_pbi_autob
 }
 
+check_zfs_ab_destroy()
+{
+	local bNum=$1
+ 	if [ -e "${PBI_TMPDIR}/${bNum}.zmnt" ] ; then
+ 		zDir=`cat ${PBI_TMPDIR}/${bNum}.zmnt`
+  		# Make sure this zfs dataset is in the PBI_APPDIR directory
+ 		echo $zDir | grep -q "${PBI_APPDIR}/" 
+	    	if [ $? -eq 0 ] ; then
+		  # Make sure all is unmounted
+		  umount -f ${PBI_CHROOTDIR}/dev >/dev/null 2>/dev/null
+		  umount -f ${PBI_CHROOTDIR}/compat/linux/proc >/dev/null 2>/dev/null
+	  	  umount -f ${PBI_CHROOTDIR}/usr/ports >/dev/null 2>/dev/null
+	 	  umount -f ${PBI_CHROOTDIR}/pkgs >/dev/null 2>/dev/null
+		  umount -f ${PBI_CHROOTDIR}/.ccache >/dev/null 2>/dev/null
+		  umount -f ${PBI_CHROOTDIR}/usr/wrkdirprefix >/dev/null 2>/dev/null
+		  sleep 3
+		  umount -f ${PBI_CHROOTDIR}/dev >/dev/null 2>/dev/null
+		  sync
+		  sleep 3
+             	  zfs destroy ${zDir}
+       	 	  rmdir ${zDir} >/dev/null 2>/dev/null
+		fi
+  	fi
+}
+
 # Start the auto-build traversal process
 do_pbi_autob() {
 
@@ -5549,117 +5589,203 @@
 	# Get this runs timestamp
 	PBI_AB_TIMESTAMP=`date | md5`
 
-	while
-	z=1
-	do
-  	  AB_FOUND="0"
-	  unset CUR_PRIORITY_PBI CUR_WORKING_PBI
-		
-	  cd "${PBI_AB_CONFDIR}"
-	  for pbi in `find . -type f -name "${PBI_CONFFILE}" | grep -v '\.svn' | sort`
-	  do
-		# Figure out the target port for this build
-		unset PBI_MAKEPORT PBI_BUILDKEY PBI_PROGVERSION PBI_REQUIRESROOT PBI_PROGNAME PBI_PROGWEB PBI_PROGAUTHOR PBI_PROGICON PBI_MKPORTBEFORE PBI_MKPORTAFTER PBI_MAKEOPTS PBI_EXCLUDELIST PBI_AB_PRIORITY PBI_HASH_EXCLUDES PBI_AB_NOTMPFS PBI_PROGREVISION
-		. ${pbi}
+	echo "`basename ${0}` started: `date`"
 
-		_cd=$(dirname $pbi | sed 's|./||')
-		PBI_CONFDIR="$_cd"
+	# Build our list of targets first
+	echo "=> Creating build queue list..."
+	ab_get_build_list
 
-		# Make sure PBI_MAKEPORT is set
-		if [ -z "${PBI_MAKEPORT}" ] ; then
-			PBI_MAKEPORT=`echo $pbi | sed 's|./||'`
-			export PBI_MAKEPORT
-		fi	
+	done=0
+	rebuildlist=0
+	while :; do
+	  activity=0
+	  # Start checking each job
+	  for build in ${JOBS-$(jot -w %02d ${PBI_AB_BUILDERS})}; do
+		# Is this builder running?
+		if [ -e "${PBI_TMPDIR}/${build}.pid" ] ; then
+			if pgrep -qF "${PBI_TMPDIR}/${build}.pid" >/dev/null 2>&1; then 
+				activity=1
+				continue; 
+			else
+				rm ${PBI_TMPDIR}/${build}.pid
+		 		# Do the ZFS destroy here if necessary
+				check_zfs_ab_destroy "$build"
+			fi
+		fi
+		if [ $done -eq 1 ] ; then continue ; fi
 
-		if [ ! -d "${PORTSDIR}/${PBI_MAKEPORT}" ] ; then 
-			echo "Skipping invalid port ${PORTSDIR}/${PBI_MAKEPORT}"
-			continue
+		# Builder idle, lets figure out the next build
+  		echo "==> Finding target for build process [$build]"
+		start_next_ab_target "$build"
+		if [ $? -eq 0 ] ; then 
+			# Nothing left to build, lets wait for any existing build to finish before exiting
+			done=1
 		fi
+		activity=1
+          done
+
+	  # Every 2 hours, we can re-scan the modules directory, catch any ones which may have been added / changed
+	  if [ $rebuildlist -eq 7200 ] ; then
+	     rebuildlist=0
+	     ab_get_build_list
+	  else
+	     rebuildlist=`expr $rebuildlist + 1`
+          fi
+
+	  # Wait before checking again
+          [ $activity -eq 1 ] && sleep 1
+	  # Nothing to do? We can end now
+          [ $activity -eq 0 ] && break
+	done 
+
+	echo "`basename ${0}` Finished: `date`"
+};
+
+ab_get_build_list()
+{
+
+   AB_FOUND="0"
+   unset CUR_PRIORITY_PBI CUR_WORKING_PBI
+
+   # Clear the tmp build list
+   rm ${PBI_TMPDIR}/.ablist 2>/dev/null
 		
-		# Check if this port can be built on this architecture
-		check_port_compat_arch "${PORTSDIR}/${PBI_MAKEPORT}"
-		if [ "$?" = "1" ] ; then
-			echo "${PBI_MAKEPORT} - Skipping for invalid system arch"
-			continue
-		fi
-		
-		# Check for missing port target
-		if [ -z "$PBI_MAKEPORT" ] ; then
-			echo "Warning: Missing PBI_MAKEPORT for ${pbi}"
-			continue
-		fi
+   cd "${PBI_AB_CONFDIR}"
+   for pbi in `find . -type f -name "${PBI_CONFFILE}" | grep -v '\.svn' | sort`
+   do
+	# Figure out the target port for this build
+	unset PBI_MAKEPORT PBI_BUILDKEY PBI_PROGVERSION PBI_REQUIRESROOT PBI_PROGNAME PBI_PROGWEB PBI_PROGAUTHOR PBI_PROGICON PBI_MKPORTBEFORE PBI_MKPORTAFTER PBI_MAKEOPTS PBI_EXCLUDELIST PBI_AB_PRIORITY PBI_HASH_EXCLUDES PBI_AB_NOTMPFS PBI_PROGREVISION
+	. ${pbi}
 
-		check_ab_needed "$PBI_MAKEPORT" "${PBI_BUILDKEY}" "$_cd" "$PBI_AB_TIMESTAMP"
-	 	if [ "$?" = "0" ] ; then	
-		   AB_FOUND="1"
+	_cd=$(dirname $pbi | sed 's|./||')
+	PBI_CONFDIR="$_cd"
 
-  	           # Unset the priority if set to 00 / 0
-	           if [ "${PBI_AB_PRIORITY}" = "00" -o "${PBI_AB_PRIORITY}" = "0" ] ; then 
-		   	unset PBI_AB_PRIORITY
-     	           fi
+	# Make sure PBI_MAKEPORT is set
+	if [ -z "${PBI_MAKEPORT}" ] ; then
+		PBI_MAKEPORT=`echo $pbi | sed 's|./||'`
+		export PBI_MAKEPORT
+	fi	
 
-		   # Check the priority of this PBI, see if it rises to the top
-		   if [ -z "${CUR_PRIORITY_PBI}" ] ; then 
-		     CUR_WORKING_PBI="${pbi}"
-		     if [ -z "$PBI_AB_PRIORITY" ] ; then 
-                       CUR_PRIORITY_PBI="$internal_ab_priority"
-	             else
-                       CUR_PRIORITY_PBI=`expr $PBI_AB_PRIORITY + 10`
-                     fi
-		     echo "Setting higher priority target: ${pbi} - Priority: ${CUR_PRIORITY_PBI}"
-                     continue
-                   fi
+	if [ ! -d "${PORTSDIR}/${PBI_MAKEPORT}" ] ; then 
+		#echo "Skipping invalid port ${PORTSDIR}/${PBI_MAKEPORT}"
+		continue
+	fi
+	
+	# Check if this port can be built on this architecture
+	check_port_compat_arch "${PORTSDIR}/${PBI_MAKEPORT}"
+	if [ "$?" = "1" ] ; then
+		#echo "${PBI_MAKEPORT} - Skipping for invalid system arch"
+		continue
+	fi
+	
+	# Check for missing port target
+	if [ -z "$PBI_MAKEPORT" ] ; then
+		#echo "Warning: Missing PBI_MAKEPORT for ${pbi}"
+		continue
+	fi
 
-		   # Bump up the supplied AB priority
-		   if [ -n "${PBI_AB_PRIORITY}" ] ; then 
-                      internal_ab_priority=`expr $PBI_AB_PRIORITY + 10`
-		   fi
+	# Check if another builder is already doing this port
+	pBuilding=0
+	for p in `ls ${PBI_TMPDIR}/*.bPort 2>/dev/null`
+	do
+		if [ "`cat $p`" = "$PBI_MAKEPORT" ] ; then
+			pBuilding=1
+			break
+		fi
+	done
+	if [ $pBuilding -eq 1 ] ; then
+		continue
+	fi
 
-		   # Check if this PBI is a higher priority
-		   if [ $CUR_PRIORITY_PBI -lt $internal_ab_priority ] ; then
-		     echo "Setting higher priority target: ${pbi} - Priority: ${internal_ab_priority}"
-		     CUR_WORKING_PBI="${pbi}"
+	check_ab_needed "$PBI_MAKEPORT" "${PBI_BUILDKEY}" "$_cd" "$PBI_AB_TIMESTAMP"
+ 	if [ "$?" = "0" ] ; then	
+	   AB_FOUND="1"
+
+ 	           # Unset the priority if set to 00 / 0
+           if [ "${PBI_AB_PRIORITY}" = "00" -o "${PBI_AB_PRIORITY}" = "0" ] ; then 
+	   	unset PBI_AB_PRIORITY
+           fi
+
+	   # Check the priority of this PBI, see if it rises to the top
+	   if [ -z "${CUR_PRIORITY_PBI}" ] ; then 
+	     CUR_WORKING_PBI="${pbi}"
+	     if [ -z "$PBI_AB_PRIORITY" ] ; then 
                      CUR_PRIORITY_PBI="$internal_ab_priority"
-		     continue
-		   fi
-		   continue
-		fi
+             else
+                     CUR_PRIORITY_PBI=`expr $PBI_AB_PRIORITY + 10`
+             fi
+	     #echo "Setting higher priority target: ${pbi} - Priority: ${CUR_PRIORITY_PBI}"
+   	     echo "${CUR_PRIORITY_PBI} $pbi" >> ${PBI_TMPDIR}/.abtmp
+             continue
+           fi
 
-	  done
+	   # Bump up the supplied AB priority
+	   if [ -n "${PBI_AB_PRIORITY}" ] ; then 
+              internal_ab_priority=`expr $PBI_AB_PRIORITY + 10`
+	   fi
 
-	  # We have something to build lets do it!
-	  if [ "$AB_FOUND" = "1" ] ; then 
-	      pbi="$CUR_WORKING_PBI"
-	      unset PBI_MAKEPORT PBI_BUILDKEY PBI_PROGVERSION PBI_REQUIRESROOT PBI_PROGNAME PBI_PROGWEB PBI_PROGAUTHOR PBI_PROGICON PBI_MKPORTBEFORE PBI_MKPORTAFTER PBI_MAKEOPTS PBI_EXCLUDELIST PBI_AB_PRIORITY PBI_HASH_EXCLUDES PBI_AB_NOTMPFS PBI_PROGREVISION
-	      . ${pbi}
+	   # Check if this PBI is a higher priority
+	   if [ $CUR_PRIORITY_PBI -lt $internal_ab_priority ] ; then
+	     #echo "Setting higher priority target: ${pbi} - Priority: ${internal_ab_priority}"
+	     CUR_WORKING_PBI="${pbi}"
+             CUR_PRIORITY_PBI="$internal_ab_priority"
+   	     echo "${internal_ab_priority} $pbi" >> ${PBI_TMPDIR}/.abtmp
+	     continue
+	   else 
+	     echo "${internal_ab_priority} $pbi" >> ${PBI_TMPDIR}/.abtmp
+	   fi
+	   continue
+  	 fi
 
-	      _cd=$(dirname $pbi | sed 's|./||')
-	      PBI_CONFDIR="$_cd"
+  done
 
-	      # Get the prog version
-	      get_pbi_progversion
+  # Sort the list
+  sort -n -r ${PBI_TMPDIR}/.abtmp > ${PBI_TMPDIR}/.ablist
+  rm ${PBI_TMPDIR}/.abtmp
+}
 
-	      if [ -z "${PBI_MAKEPORT}" ] ; then
-	 	PBI_MAKEPORT=$(dirname $pbi | sed 's|./||')
-	      fi	
+start_next_ab_target()
+{
+  local curBuilder="$1"
+  # No list to parse?
+  if [ ! -e "${PBI_TMPDIR}/.ablist" ] ; then return 0; fi
 
-	      echo "Starting build of: $PBI_MAKEPORT  - Priority: $CUR_PRIORITY_PBI"
+  # Get the last on the list
+  CUR_WORKING_PBI=`cat ${PBI_TMPDIR}/.ablist | head -1 | cut -d ' ' -f 2`
+  if [ -z "${CUR_WORKING_PBI}" ] ; then return 0; fi
 
-	      # Start the build now
-	      start_ext_ab "$PBI_MAKEPORT" \
-		"${PBI_BUILDKEY}" "${PBI_PROGVERSION}" \
-		"${_cd}" "${PBI_AB_OUTDIR}" "${PBI_AB_TIMESTAMP}"
+  cd "${PBI_AB_CONFDIR}"
 
-	      echo " "
+  # We have something to build lets do it!
+  pbi="$CUR_WORKING_PBI"
+  unset PBI_MAKEPORT PBI_BUILDKEY PBI_PROGVERSION PBI_REQUIRESROOT PBI_PROGNAME PBI_PROGWEB PBI_PROGAUTHOR PBI_PROGICON PBI_MKPORTBEFORE PBI_MKPORTAFTER PBI_MAKEOPTS PBI_EXCLUDELIST PBI_AB_PRIORITY PBI_HASH_EXCLUDES PBI_AB_NOTMPFS PBI_PROGREVISION
+  . ${pbi}
 
-	  else
-	    # If no builds left, we are done!
-            break
-          fi
-	done 
-	
-}
+  _cd=$(dirname $pbi | sed 's|./||')
+  PBI_CONFDIR="$_cd"
 
+  # Get the prog version
+  get_pbi_progversion
+
+  if [ -z "${PBI_MAKEPORT}" ] ; then
+     PBI_MAKEPORT=$(dirname $pbi | sed 's|./||')
+  fi	
+
+  echo "==> Starting build process [$curBuilder]: $PBI_MAKEPORT"
+  echo "$PBI_MAKEPORT" >${PBI_TMPDIR}/${curBuilder}.bPort
+
+  # Remove this from the queue
+  cat ${PBI_TMPDIR}/.ablist | tail -n +2 > ${PBI_TMPDIR}/.abtmp
+  mv ${PBI_TMPDIR}/.abtmp ${PBI_TMPDIR}/.ablist
+
+  # Start the build now
+  start_ext_ab "$PBI_MAKEPORT" \
+  "${PBI_BUILDKEY}" "${PBI_PROGVERSION}" \
+  "${_cd}" "${PBI_AB_OUTDIR}" "${PBI_AB_TIMESTAMP}" "${curBuilder}"
+
+  return 1
+};
+
 # Prune any outdir files which don't have matching modules
 do_pbi_autob_prune() {
 	if [ "${PBI_AB_PRUNE}" != "YES" ] ; then return 0 ; fi
@@ -5689,6 +5815,7 @@
 	_cd="${4}"
 	_od="${5}/${_cd}"
 	local _abkey="$6"
+	local eBuilder="$7"
 	_flags=""
 	_flags="-c ${_cd} -d ${PORTSDIR} -o ${_od} --delbuild"
 	if [ -n "${PBI_AB_SSLPRIVKEY}" ] ; then
@@ -5716,7 +5843,7 @@
 	get_pbi_progversion
 
 	#echo "Starting build of ${_mp} - ${_pv}"
-	echo "pbi_makeport ${_flags} ${_mp}"
+	#echo "pbi_makeport ${_flags} ${_mp}"
 
 	if [ ! -d "${_od}" ] ; then mkdir -p "${_od}" ; fi
 
@@ -5744,55 +5871,71 @@
 	# Move old PBIs to archived folder
         oldVersion=`cat ${_od}/pbi-version 2>/dev/null`
 	if [ "$oldVersion" != "$PBI_PROGVERSION" ]; then
-		echo "Archiving old PBIs..."
+		#echo "Archiving old PBIs..."
 		archive_old_pbis "${_od}" "$PBI_AB_ARCHIVENUM"
 	fi
 
+	# Set some variables we can call in pbi_makeport
+	PBI_AB_TMPDIR="${PBI_TMPDIR}"
+	PBI_AB_BUILDER="$eBuilder"
+	export PBI_AB_TMPDIR PBI_AB_BUILDER
+
 	# Add some header info to log file
 	echo "Starting build: `date`" >${_od}/build.log
 	echo "Build Command:" >>${_od}/build.log
 	echo "pbi_makeport ${_flags} ${_mp}" >>${_od}/build.log
 	echo "------------------------------------------------------" >>${_od}/build.log
+	
+	# Create the build script we will run
+	cat >${PBI_TMPDIR}/${eBuilder}.sh << EOF
+#!/bin/sh
 
-	# Start the build now
-	pbi_makeport ${_flags} ${_mp} >>${_od}/build.log 2>>${_od}/build.log
-	if [ "$?" = "0" ] ; then
+# Start the build now
+exec pbi_makeport ${_flags} ${_mp}
+if [ "$?" = "0" ] ; then
 
-		echo "$PBI_PROGVERSION" > "${_od}/pbi-version"
-		echo "OK" > "${_od}/pbi-result"
+	echo "$PBI_PROGVERSION" > "${_od}/pbi-version"
+	echo "OK" > "${_od}/pbi-result"
 
-		# Save the mdate file
-		date "+%Y%m%d %H%M%S" >${_od}/pbi-mdate
+	# Save the mdate file
+	date "+%Y%m%d %H%M%S" >${_od}/pbi-mdate
 
-		if [ -n "${PBI_AB_HELPS}" ] ; then
-			${PBI_AB_HELPS} "OK" "${_od}"
-		fi
+	if [ -n "${PBI_AB_HELPS}" ] ; then
+		${PBI_AB_HELPS} "OK" "${_od}"
+	fi
 
-		# Copy over a description file
-		if [ -e "${PORTSDIR}/${_mp}/pkg-descr" ] ; then
-			cp "${PORTSDIR}/${_mp}/pkg-descr" "${_od}/pbi-descr"
-		fi
-		if [ -e "${_cd}/pbi-descr" ] ; then
-			cp "${_cd}/pbi-descr" "${_od}/pbi-descr"
-		fi
+	# Copy over a description file
+	if [ -e "${PORTSDIR}/${_mp}/pkg-descr" ] ; then
+		cp "${PORTSDIR}/${_mp}/pkg-descr" "${_od}/pbi-descr"
+	fi
+	if [ -e "${_cd}/pbi-descr" ] ; then
+		cp "${_cd}/pbi-descr" "${_od}/pbi-descr"
+	fi
 
-		# Generate patch files to the new version of this PBI
-		if [ "$PBI_AB_GENPATCH" = "YES" -a -d "${_od}/archived" ] ; then
-			gen_pbi_patches "${_od}" "${_od}/archived" >>${_od}/build.log 2>>${_od}/build.log
-		fi
+	# Generate patch files to the new version of this PBI
+	if [ "$PBI_AB_GENPATCH" = "YES" -a -d "${_od}/archived" ] ; then
+		gen_pbi_patches "${_od}" "${_od}/archived" >>${_od}/build.log 2>>${_od}/build.log
+	fi
 
-		rm "${_od}/build.log.bz2" >/dev/null 2>/dev/null
-		bzip2 "${_od}/build.log"
+	rm "${_od}/build.log.bz2" >/dev/null 2>/dev/null
+	bzip2 "${_od}/build.log"
 
-	else
-		# Save the md5sum of the ports directory
-		tar cvf - -C "${PORTSDIR}/${_mp}" . 2>/dev/null | md5 -q >${_od}/.failed-csum
+else
+	# Save the md5sum of the ports directory
+	tar cvf - -C "${PORTSDIR}/${_mp}" . 2>/dev/null | md5 -q >${_od}/.failed-csum
 
-		echo "Failed" > "${_od}/pbi-result"
-		if [ -n "${PBI_AB_HELPS}" ] ; then
-			${PBI_AB_HELPS} "FAILED" "${_od}"
-		fi
+	echo "Failed" > "${_od}/pbi-result"
+	if [ -n "${PBI_AB_HELPS}" ] ; then
+		${PBI_AB_HELPS} "FAILED" "${_od}"
 	fi
+fi
+EOF
+
+  chmod 755 ${PBI_TMPDIR}/${eBuilder}.sh
+  (${PBI_TMPDIR}/${eBuilder}.sh >>${_od}/build.log 2>>${_od}/build.log) &
+  echo "$!" > ${PBI_TMPDIR}/${eBuilder}.pid
+  return
+
 }
 
 # Function which begins to generate patch files from archived PBIs to current
@@ -6232,7 +6375,7 @@
 	if [ -e "${PBI_AB_OUTDIR}/${_cd}/pbi-buildkey" ] ; then
 		if [ "`cat ${PBI_AB_OUTDIR}/${_cd}/pbi-buildkey`" != "$_bk" \
 			 -a -n "${_bk}" ]; then 
-                   echo "$_port BUILDKEY bump, rebuild triggered."
+                   #echo "$_port BUILDKEY bump, rebuild triggered."
                    internal_ab_priority="9"
                    return 0
                 fi
@@ -6242,7 +6385,7 @@
 	if [ -e "${PBI_AB_OUTDIR}/${_cd}/.abkey" -a -e "${PBI_AB_OUTDIR}/${_cd}/pbi-result" ] ; then
 	  if [ "`cat ${PBI_AB_OUTDIR}/${_cd}/.abkey`" = "$_abkey" ] ; then
 	    if [ "`cat ${PBI_AB_OUTDIR}/${_cd}/pbi-result`" != "OK" ] ; then
-              echo "$_port - Skipping failed from this run"
+              #echo "$_port - Skipping failed from this run"
 	      return 1
   	    fi
   	  fi
@@ -6255,7 +6398,7 @@
 		_fcsum="`cat ${PBI_AB_OUTDIR}/${_cd}/.failed-csum`"
 		_ncsum="`tar cvf - -C "${PORTSDIR}/${_port}" . 2>/dev/null | md5 -q`"
 		if [ "$_fcsum" = "$_ncsum" ] ; then
-            		echo "$_port - Skipping failed"
+            		#echo "$_port - Skipping failed"
 			return 1
 		fi
  	fi	
@@ -6263,7 +6406,7 @@
 	# See if we have an existing PBI
 	ls ${PBI_AB_OUTDIR}/${_cd}/*.pbi >/dev/null 2>/dev/null
 	if [ "${?}" != "0" ]; then 
-	  echo "$_port - No existing PBI"
+	  #echo "$_port - No existing PBI"
           internal_ab_priority="8"
           return 0 
         fi
@@ -6271,7 +6414,7 @@
 	# See if we have a saved version
 	if [ ! -e "${PBI_AB_OUTDIR}/${_cd}/pbi-version" ]; then 
 		#echo "No saved pbi-version"
-	  	echo "$_port - No existing version"
+	  	#echo "$_port - No existing version"
           	internal_ab_priority="7"
 		return 0
 	fi
@@ -6286,11 +6429,11 @@
 		if [ "$PORTVER" = "$oldPortVer" ] ; then
 			# Just a minor portrev bump
           		internal_ab_priority="2"
-			echo "$_port revision bump: $oldVersion -> $PBI_PROGVERSION"
+			#echo "$_port revision bump: $oldVersion -> $PBI_PROGVERSION"
 		else
 			# Real version change
           		internal_ab_priority="3"
-			echo "$_port version bump: $oldVersion -> $PBI_PROGVERSION"
+			#echo "$_port version bump: $oldVersion -> $PBI_PROGVERSION"
 		fi
 		return 0
 	fi
@@ -6311,6 +6454,9 @@
 
 do_pbi_make() {
 
+	# See if we need to enable pkgng
+	detect_pkgng
+
   	# Load the PBI settings
 	get_pbi_progversion
 	get_pbi_progname
@@ -6480,14 +6626,22 @@
 	umount -f ${PBI_CHROOTDIR}/.ccache >/dev/null 2>/dev/null
 	umount -f ${PBI_CHROOTDIR}/usr/wrkdirprefix >/dev/null 2>/dev/null
 
+	# Sleep a moment before we try this again, seems it takes a moment to clear up
+	sleep 2
+	umount -f ${PBI_CHROOTDIR}/dev >/dev/null 2>/dev/null
+
 	if [ "${PBI_KEEPBUILDFILES}" = "YES" ] ; then return ; fi
 
 	# Cleanup ZFS dataset
         isDirZFS "${PBI_CHROOTDIR}" "1"
         if [ $? -eq 0 ] ; then
    	     tank=`getZFSTank "$PBI_CHROOTDIR"`
-             zfs destroy ${tank}${PBI_CHROOTDIR}
-  	     rmdir ${PBI_CHROOTDIR} >/dev/null 2>/dev/null
+	     sleep 1
+	     # If we are running as a result of pbi_autobuild, let it do the ZFS cleanup
+	     if [ -z "${PBI_AB_TMPDIR}" ] ; then
+             	zfs destroy ${tank}${PBI_CHROOTDIR}
+  	     	rmdir ${PBI_CHROOTDIR} >/dev/null 2>/dev/null
+	     fi
 	     return
         fi
 
@@ -6526,6 +6680,9 @@
 	if [ -n "$PBI_CHROOTZFS" ] ; then
            tank=`getZFSTank "$PBI_CHROOTZFS"`
 	   echo "Cloning ${PBI_CHROOTZFS} -> ${PBI_CHROOTDIR}"
+	   if [ -n "${PBI_AB_TMPDIR}" ] ; then
+              echo "${tank}${PBI_CHROOTDIR}" > ${PBI_AB_TMPDIR}/${PBI_AB_BUILDER}.zmnt
+	   fi
            zfs clone ${tank}${PBI_CHROOTZFS}@clean ${tank}${PBI_CHROOTDIR}
 	   if [ $? -ne 0 ] ; then exit_err "Failed creating clean ZFS base snapshot"; fi
 	else
@@ -6943,12 +7100,29 @@
 
 # Cleanup after caught exit
 exit_trap() {
+	echo "Cleaning up..."
 	# If a download is running, kill it
 	if [ -n  "${FETCH_PID}" ] ; then 
 	   echo "Killing ${FETCH_PID}"
 	   kill -s INT ${FETCH_PID}
            sleep 2
 	fi
+
+	# If we are running as an auto-build service we need to cleanup
+	if [ "`basename ${0}`" = "pbi_autobuild" ] ; then
+	   for j in `ls ${PBI_TMPDIR}/*.pid 2>/dev/null`
+	   do
+	      bNum=`basename $j | cut -d '.' -f 1 `
+	      if pgrep -qF "${j}" >/dev/null 2>&1; then 
+	         echo "===>Cleaning up processes [$bNum]"
+		 pkill -9 -P `cat ${j}`
+ 		 sleep 4 
+		 # Do the ZFS destroy here, since running zfs destroy concurrently as we cleanup can cause a panic
+		 check_zfs_ab_destroy "$bNum"
+	      fi
+	   done
+	fi
+
         chroot_make_cleanup
 	rm_pbipatchfiles
 	rm_tmpdir



More information about the Commits mailing list