[PC-BSD Commits] r19619 - pcbsd/current/src-sh/pbi-manager

svn at pcbsd.org svn at pcbsd.org
Tue Oct 9 11:48:19 PDT 2012


Author: kris
Date: 2012-10-09 18:48:19 +0000 (Tue, 09 Oct 2012)
New Revision: 19619

Modified:
   pcbsd/current/src-sh/pbi-manager/pbi-manager
Log:

Large update to pbi-manager

Make it ZFS aware, now if we have /usr/pbi as a ZFS dataset, we will do
some ZFS-kung-fu to make creating build environments instant.

Also some additional speedups while here



Modified: pcbsd/current/src-sh/pbi-manager/pbi-manager
===================================================================
--- pcbsd/current/src-sh/pbi-manager/pbi-manager	2012-10-09 16:22:09 UTC (rev 19618)
+++ pcbsd/current/src-sh/pbi-manager/pbi-manager	2012-10-09 18:48:19 UTC (rev 19619)
@@ -1510,13 +1510,6 @@
 # rm tmpdir
 rm_buildfiles() {
 	if [ "${PBI_KEEPBUILDFILES}" = "YES" ] ; then return ; fi
-	if [ -z "${PBI_PROGDIRPATH}" ] ; then return ; fi
-	if [ "`basename $0`" = "pbi_makeport_chroot" -a -d "${PBI_PROGDIRPATH}" ] ; then 
-		echo "Cleaning ${PBI_PROGDIRPATH}"
-		rm -rf "${PBI_PROGDIRPATH}" >/dev/null 2>/dev/null
-		chflags -R noschg "${PBI_PROGDIRPATH}" >/dev/null 2>/dev/null
-		rm -rf "${PBI_PROGDIRPATH}" >/dev/null 2>/dev/null
-	fi
 	if [ -z "$PBI_CHROOTDIR" ] ; then return ; fi
 	chroot_make_cleanup
 }
@@ -1680,7 +1673,6 @@
 	echo "NO_IGNORE=yes" >> ${MAKE_CONF}
 	echo "PACKAGE_BUILDING=yes" >> ${MAKE_CONF}
 
-	mkdir /usr/wrkdirprefix
 	echo "WRKDIRPREFIX=/usr/wrkdirprefix" >> ${MAKE_CONF}
 	echo "DEPENDS_CLEAN=YES" >> ${MAKE_CONF}
 
@@ -3472,9 +3464,13 @@
 	get_pbi_progname
 	echo "Creating PBI: ${PBI_PROGNAME}-${PBI_PROGVERSION}"
 
-	mk_header_dir
-	mk_stage_dir
-	
+
+	if [ "`basename $0`" = "pbi_makeport" -o "`basename $0`" = "pbi_makeport_chroot" ] ; then
+	  PBI_STAGEDIR="${PBI_PROGDIRPATH}"
+	else
+	  mk_stage_dir
+	fi
+
 	copy_resource_dir
 	clean_stage_dir
 
@@ -3489,11 +3485,14 @@
 	mk_hash_list
 
 	mk_archive_file
+
+	mk_header_dir
  	save_pbi_details_to_header
 	mk_header_file
+	rm_header_dir
+
 	mk_output_pbi
 
-	rm_header_dir
 	rm_stage_dir
 }
 
@@ -4059,10 +4058,21 @@
 
 # Start creating the application archive 
 mk_archive_file() {
+        # Build module list of excludes
+        if [ -n "$PBI_EXCLUDELIST" ] ; then
+                for excl in $PBI_EXCLUDELIST
+                do
+                        if [ -z "$_excOpts" ] ; then
+                                _excOpts="--exclude ${excl}"
+                        else
+                                _excOpts="$_excOpts --exclude ${excl}"
+                        fi
+                done
+        fi
 	PBI_CREATE_ARCHIVE="${PBI_CREATE_OUTDIR}/.PBI.$$.tbz"
 	if test_tar_lzma ; then _tcmp="J" ; else _tcmp="j" ; fi
 	echo "Creating compressed archive..."
-	tar cv${_tcmp}f "${PBI_CREATE_ARCHIVE}" -C ${PBI_STAGEDIR} . 2>/dev/null 
+	tar cv${_tcmp}f "${PBI_CREATE_ARCHIVE}" ${_excOpts} -C ${PBI_STAGEDIR} . 2>/dev/null 
 }
 
 # Start creating the header archive
@@ -6468,9 +6478,20 @@
 	umount -f ${PBI_CHROOTDIR}/usr/ports >/dev/null 2>/dev/null
 	umount -f ${PBI_CHROOTDIR}/pkgs >/dev/null 2>/dev/null
 	umount -f ${PBI_CHROOTDIR}/.ccache >/dev/null 2>/dev/null
-	umount -f ${PBI_CHROOTDIR} >/dev/null 2>/dev/null
+	umount -f ${PBI_CHROOTDIR}/usr/wrkdirprefix >/dev/null 2>/dev/null
 
 	if [ "${PBI_KEEPBUILDFILES}" = "YES" ] ; then return ; fi
+
+	# Cleanup ZFS dataset
+        isDirZFS "${PBI_CHROOTDIR}" "1"
+        if [ $? -eq 0 ] ; then
+   	     tank=`getZFSTank "$PBI_CHROOTDIR"`
+             zfs destroy ${tank}${PBI_CHROOTDIR}
+  	     rmdir ${PBI_CHROOTDIR} >/dev/null 2>/dev/null
+	     return
+        fi
+
+	# Cleanup normal directory
 	rm -rf "${PBI_CHROOTDIR}" >/dev/null 2>/dev/null
 	chflags -R noschg ${PBI_CHROOTDIR} >/dev/null 2>/dev/null
 	rm -rf "${PBI_CHROOTDIR}" >/dev/null 2>/dev/null
@@ -6479,9 +6500,17 @@
 # Function which extracts the clean chroot environment for the PBI
 chroot_extract() {
 
-	# If no chroot file exists, make it first
-	PBI_CHROOTFILE="${PBI_APPDIR}/.pbi-world-$ARCH.txz"
-	[ -e "${PBI_CHROOTFILE}" ] || mk_chroot_file
+	# If no freebsd base exists, make it first
+	isDirZFS "${PBI_APPDIR}"
+	if [ $? -eq 0 ] ; then
+		# Use ZFS base for cloning
+		PBI_CHROOTZFS="${PBI_APPDIR}/.pbi-world-$ARCH"
+		[ -e "${PBI_CHROOTZFS}/COPYRIGHT" ] || mk_chroot_file
+	else
+		# Use regular .txz file
+		PBI_CHROOTFILE="${PBI_APPDIR}/.pbi-world-$ARCH.txz"
+		[ -e "${PBI_CHROOTFILE}" ] || mk_chroot_file
+	fi
 
 	# Set the chroot path
 	PBI_CHROOTDIR="${PBI_PROGDIRPATH}.chroot"
@@ -6493,19 +6522,27 @@
 	# Create the new chroot dir
 	mkdir -p "${PBI_CHROOTDIR}"
 
+	# If on ZFS, we can just clone our existing base system
+	if [ -n "$PBI_CHROOTZFS" ] ; then
+           tank=`getZFSTank "$PBI_CHROOTZFS"`
+	   echo "Cloning ${PBI_CHROOTZFS} -> ${PBI_CHROOTDIR}"
+           zfs clone ${tank}${PBI_CHROOTZFS}@clean ${tank}${PBI_CHROOTDIR}
+	   if [ $? -ne 0 ] ; then exit_err "Failed creating clean ZFS base snapshot"; fi
+	else
+	   echo "Extracting chroot environment..."
+	   tar xvf ${PBI_CHROOTFILE} -C "${PBI_CHROOTDIR}" >/dev/null 2>/dev/null
+	   [ $? -ne 0 ] && exit_err "Failed extracting chroot environment!"
+	fi
+
 	# If we plan on using TMPFS mount it now
+	mkdir -p ${PBI_CHROOTDIR}/usr/wrkdirprefix
 	if [ "$MKTMPFS" = "YES" ] ; then 
-		mount -t tmpfs tmpfs ${PBI_CHROOTDIR}
+		mount -t tmpfs tmpfs ${PBI_CHROOTDIR}/usr/wrkdirprefix
 	fi
 
-	echo "Extracting chroot environment..."
-	tar xvf ${PBI_CHROOTFILE} -C "${PBI_CHROOTDIR}" >/dev/null 2>/dev/null
-	[ $? -ne 0 ] && exit_err "Failed extracting chroot environment!"
-
 	# Copy resolv.conf
 	cp /etc/resolv.conf ${PBI_CHROOTDIR}/etc/resolv.conf
 
-
 	# Copy our binary wrapper
 	mkdir ${PBI_CHROOTDIR}${PBI_APPDIR} 2>/dev/null
 	cp ${PBI_WRAPPERFILE} ${PBI_CHROOTDIR}${PBI_WRAPPERFILE}
@@ -6618,11 +6655,37 @@
  	  echo "Fetching FreeBSD chroot environment... This may take a while..."
   	  fetch -o rel-$ARCH.txz ${MIRRORURL}/${SYSVER}/${ARCH}/netinstall/fbsd-release.txz
     	  fetch -o rel-$ARCH.md5 ${MIRRORURL}/${SYSVER}/${ARCH}/netinstall/fbsd-release.txz.md5
+  	  fetch -o src-$ARCH.txz ${MIRRORURL}/${SYSVER}/${ARCH}/netinstall/extras/components/src.txz
 
     	  [ `md5 -q rel-$ARCH.txz` != `cat rel-$ARCH.md5` ] && exit_err "Error in download data, checksum mismatch.. Please try again later."
 
-	  mv rel-$ARCH.txz ${PBI_CHROOTFILE}
-	  rm rel-$ARCH.md5
+	  isDirZFS "${PBI_APPDIR}"
+          if [ $? -eq 0 ] ; then   
+            # Use ZFS base for cloning
+	    echo "Creating ZFS ${PBI_CHROOTZFS} dataset..."
+            tank=`getZFSTank "$PBI_APPDIR"`
+	    isDirZFS "${PBI_CHROOTZFS}" "1"
+            if [ $? -ne 0 ] ; then   
+              zfs create -o mountpoint=${PBI_CHROOTZFS} -p ${tank}${PBI_CHROOTZFS}
+	      if [ $? -ne 0 ] ; then exit_err "Failed creating ZFS base dataset"; fi
+	    fi
+	    echo "Extracting rel-$ARCH.txz..."
+	    tar xvpf rel-$ARCH.txz -C ${PBI_CHROOTZFS} 2>/dev/null
+	    if [ $? -ne 0 ] ; then exit_err "Failed extracting ZFS base dataset"; fi
+	    mkdir -p ${PBI_CHROOTZFS}/usr/src 2>/dev/null
+	    echo "Extracting src-$ARCH.txz..."
+	    tar xvpf src-$ARCH.txz -C ${PBI_CHROOTZFS}/usr/src 2>/dev/null
+	    if [ $? -ne 0 ] ; then exit_err "Failed extracting ZFS base src dataset"; fi
+	    rm rel-$ARCH.txz
+	    rm src-$ARCH.txz
+	    rm rel-$ARCH.md5
+            zfs snapshot ${tank}${PBI_CHROOTZFS}@clean
+	    if [ $? -ne 0 ] ; then exit_err "Failed creating clean ZFS base snapshot"; fi
+	  else
+	    mv rel-$ARCH.txz ${PBI_CHROOTFILE}
+	    rm rel-$ARCH.md5
+	  fi
+
 	  return
  	fi
 
@@ -6705,7 +6768,7 @@
 	echo "Running buildworld / installworld (into a chroot)"
 	touch ${PBI_BUILDSRC}/Makefile
 	cd ${PBI_BUILDSRC}
-        make ${PBI_BUILDFLAGS} buildworld >>${PBI_BUILDLOG} 2>>${PBI_BUILDLOG}
+        make ${PBI_BUILDFLAGS} TARGET=$ARCH buildworld >>${PBI_BUILDLOG} 2>>${PBI_BUILDLOG}
         if [ "$?" != "0" ] ; then
                 cd
                 if [ "${PBI_BUILDSRC}" != "/usr/src" -a "${PBI_DELETE_BUILD}" != "0" ] ; then
@@ -6714,7 +6777,22 @@
                 fi
                 exit_err "Buildworld failed! Logfile saved: ${PBI_BUILDLOG}"
         fi
-        make ${PBI_BUILDFLAGS} installworld DESTDIR=${PBI_BUILDTARGET} >>${PBI_BUILDLOG} 2>>${PBI_BUILDLOG}
+
+	# See if we need to create a ZFS dataset
+	isDirZFS "${PBI_APPDIR}"
+        if [ $? -eq 0 ] ; then   
+            # Use ZFS base for cloning
+	    echo "Creating ZFS ${PBI_CHROOTZFS} dataset..."
+            tank=`getZFSTank "$PBI_APPDIR"`
+	    isDirZFS "${PBI_CHROOTZFS}" "1"
+            if [ $? -ne 0 ] ; then   
+              zfs create -o mountpoint=${PBI_CHROOTZFS} -p ${tank}${PBI_CHROOTZFS}
+	      if [ $? -ne 0 ] ; then exit_err "Failed creating ZFS base dataset"; fi
+	    fi
+	    PBI_BUILDTARGET="$PBI_CHROOTZFS"
+	fi
+
+        make ${PBI_BUILDFLAGS} installworld TARGET=$ARCH DESTDIR=${PBI_BUILDTARGET} >>${PBI_BUILDLOG} 2>>${PBI_BUILDLOG}
         if [ "$?" != "0" ] ; then
                 cd
                 if [ "${PBI_BUILDSRC}" != "/usr/src" -a "${PBI_DELETE_BUILD}" != "0" ] ; then
@@ -6723,7 +6801,7 @@
                 fi
                 exit_err "Buildworld failed! Logfile saved: ${PBI_BUILDLOG}"
         fi
-        make ${PBI_BUILDFLAGS} distribution DESTDIR=${PBI_BUILDTARGET} >>${PBI_BUILDLOG} 2>>${PBI_BUILDLOG}
+        make ${PBI_BUILDFLAGS} distribution TARGET=$ARCH DESTDIR=${PBI_BUILDTARGET} >>${PBI_BUILDLOG} 2>>${PBI_BUILDLOG}
         if [ "$?" != "0" ] ; then
                 cd
                 if [ "${PBI_BUILDSRC}" != "/usr/src" -a "${PBI_DELETE_BUILD}" != "0" ] ; then
@@ -6738,6 +6816,14 @@
 	mkdir -p ${PBI_BUILDTARGET}/usr/src >/dev/null 2>/dev/null
 	tar cvf - -C "${PBI_BUILDSRC}" --exclude "\.svn/" . 2>/dev/null | tar xvf - -C "${PBI_BUILDTARGET}/usr/src" 2>/dev/null
 
+	# If using ZFS we can stop here
+	if [ -n "$PBI_CHROOTZFS" ] ; then
+           	zfs snapshot ${tank}${PBI_CHROOTZFS}@clean
+	    	if [ $? -ne 0 ] ; then exit_err "Failed creating clean ZFS base snapshot"; fi
+		rm ${PBI_BUILDLOG}
+		return
+	fi
+
 	echo "Creating chroot environment tarball"
 	tar cvjf ${PBI_CHROOTFILE} -C ${PBI_BUILDTARGET} . >>${PBI_BUILDLOG} 2>>${PBI_BUILDLOG}
 	if [ $? -ne 0 ] ; then
@@ -6769,12 +6855,12 @@
 
 	# Define some commands
      	if [ $PKGNG -eq 1 ] ; then
-		pkgInf="pkg info -L"
+		pkgInf="pkg info -l"
      	else
 		pkgInf="pkg_info -L"
 	fi
 
- 	$pkgInf ${_pname} | sed "s|^${PBI_PROGDIRPATH}/||g" \
+ 	${pkgInf} ${_pname} | sed "s|^${PBI_PROGDIRPATH}/||g" \
 		| grep -v -e "^Information for" -e "^Files:" -e "owns the following" \
 		| tr -s '\t' ' ' \
 		| tr -d ' ' \
@@ -7037,7 +7123,46 @@
   fi
 };
 
+# Check if the target directory is on ZFS
+# Arg1 = The dir to check
+# Arg2 = If set to 1, don't dig down to lower level directory
+isDirZFS() {
+  local _chkDir="$1"
+  while :
+  do
+     # Is this dir a ZFS mount
+     mount | grep -w "on $_chkDir " | grep -qw "(zfs," && return 0
+        
+     # Quit if not walking down
+     if [ "$2" = "1" ] ; then return 1 ; fi
+        
+     if [ "$_chkDir" = "/" ] ; then break ; fi
+     _chkDir=`dirname $_chkDir`
+  done
+ 
+  return 1
+}
 
+# Get the ZFS tank name for a directory
+# Arg1 = Directory to check
+getZFSTank() {
+  local _chkDir="$1"
+  while :
+  do
+     line=`mount | grep -w -e $_chkDir -e "(zfs,"`
+     mount | grep -qw -e $_chkDir -e "(zfs,"
+     if [ $? -eq 0 ] ; then
+        echo $line | cut -d '/' -f -1 | awk '{print $1}'
+        return 0
+     fi
+
+     if [ "$_chkDir" = "/" ] ; then return 1 ; fi
+     _chkDir=`dirname $_chkDir`
+  done
+
+  return 1
+}
+
 # Main program operation
 ##############################################################
 init_vars



More information about the Commits mailing list