[PC-BSD Commits] r21848 - in pcbsd/current/src-sh: libsh pbi-manager pc-thinclient pc-updatemanager port-files warden/scripts/backend xtrafiles/local/share/pcbsd xtrafiles/local/share/pcbsd/conf

svn at pcbsd.org svn at pcbsd.org
Wed Mar 13 20:07:55 PDT 2013


Author: kris
Date: 2013-03-14 03:07:55 +0000 (Thu, 14 Mar 2013)
New Revision: 21848

Added:
   pcbsd/current/src-sh/xtrafiles/local/share/pcbsd/conf/
   pcbsd/current/src-sh/xtrafiles/local/share/pcbsd/conf/pcbsd-mirrors
Modified:
   pcbsd/current/src-sh/libsh/functions.sh
   pcbsd/current/src-sh/pbi-manager/pbi-manager
   pcbsd/current/src-sh/pc-thinclient/pc-thinclient
   pcbsd/current/src-sh/pc-updatemanager/pc-updatemanager
   pcbsd/current/src-sh/port-files/Makefile
   pcbsd/current/src-sh/port-files/pkg-plist
   pcbsd/current/src-sh/warden/scripts/backend/functions.sh
Log:

Huge overhaul of how we download files from mirrors. Applies to
PBIs, Thin-Client, Warden, package updates and more! 

Welcome the new aria2 (where have you been all my life) downloader

For shell scripts, we now have a new function: 

get_file_from_mirrors()

This will use the "aria2c" binary to download the specified file from the
PC-BSD mirrors. It will pull chunks from multiple mirrors, track the speeds
of mirrors, auto-fail-over if a mirror goes down, wash your dishes and more!

Since it can pull from all the fastest mirrors to your location,
users should see significant download speed improvements. Downloading a
3.4GB ISO took me over an hour before, using aria2 it took about 10 minutes. 

Testing and feedback welcome!



Modified: pcbsd/current/src-sh/libsh/functions.sh
===================================================================
--- pcbsd/current/src-sh/libsh/functions.sh	2013-03-13 17:51:24 UTC (rev 21847)
+++ pcbsd/current/src-sh/libsh/functions.sh	2013-03-14 03:07:55 UTC (rev 21848)
@@ -49,6 +49,106 @@
   return 
 }
 
+# Function which returns the installed list of PC-BSD mirrors for use
+# with the aria2c command
+# Will return just a single mirror, if the user has manually specified one
+# in /usr/local/etc/pcbsd.conf
+get_aria_mirror_list()
+{
+  if [ -z $1 ] ; then
+     exit_err "Need to supply file to grab from mirrors..."
+  fi
+
+  # Set the mirror URL
+  local VAL="`cat ${PCBSD_ETCCONF} 2>/dev/null | grep 'PCBSD_MIRROR: ' | sed 's|PCBSD_MIRROR: ||g'`"
+  if [ -n "$VAL" ] ; then
+     echo "${VAL}${1}"
+     return
+  fi
+
+  if [ ! -e "/usr/local/share/pcbsd/conf/pcbsd-mirrors" ] ; then
+     exit_err "Missing mirror list: /usr/local/share/pcbsd/conf/pcbsd-mirrors"
+  fi
+
+  # Build the mirror list
+  while read line
+  do
+    VAL="$VAL ${line}${1}"
+  done < /usr/local/share/pcbsd/conf/pcbsd-mirrors
+  echo ${VAL}
+}
+
+# Function to download a file from the pcbsd mirrors
+# Arg1 = Remote File URL
+# Arg2 = Where to save file
+get_file_from_mirrors()
+{
+   _rf="${1}"
+   _lf="${2}"
+
+   # Get any proxy information
+   . /etc/profile
+
+   # Split up the dir / file name
+   local aDir=`dirname $_lf`
+   local aFile=`basename $_lf`
+
+   # Server status flag
+   local aStatFile=${HOME}/.pcbsd-aria-stat
+   if [ -e "$aStatFile" ] ; then
+     local aStat="--server-stat-of=$aStatFile --server-stat-if=$aStatFile --uri-selector=adaptive --server-stat-timeout=864000"
+   else
+     local aStat="--server-stat-of=$aStatFile --uri-selector=adaptive"
+   fi
+   touch $aStatFile
+
+   # Get mirror list
+   local mirrorList="$(get_aria_mirror_list $1)"
+   
+   # Running from a non GUI?
+   if [ "$GUI_FETCH_PARSING" != "YES" -a "$PBI_FETCH_PARSING" != "YES" -a -z "$PCFETCHGUI" ] ; then
+      aria2c -k 5M ${aStat} --check-certificate=false --file-allocation=none -d "${aDir}" -o "${aFile}" ${mirrorList}
+      return $?
+   fi
+
+   echo "FETCH: ${_rf}"
+
+   # Doing a front-end download, parse the output of fetch
+   _eFile="/tmp/.fetch-exit.$$"
+   fetch -s "`echo ${mirrorList} | awk '{print $1}'`" > /tmp/.fetch-size.$$ 2>/dev/null
+   _fSize=`cat /tmp/.fetch-size.$$ 2>/dev/null`
+   _fSize="`expr ${_fSize} / 1024 2>/dev/null`"
+   rm "/tmp/.fetch-size.$$" 2>/dev/null
+   _time=1
+
+   ( aria2c -o ${aFile} -d ${aDir} -k 5M ${aStat} --check-certificate=false --file-allocation=none ${mirrorList} >/dev/null 2>/dev/null ; echo "$?" > ${_eFile} ) &
+   FETCH_PID=`ps -auwwwx | grep -v grep | grep "aria2c -o ${aFile}" | awk '{print $2}'`
+   while : 
+   do
+      if [ -e "${_lf}" ] ; then
+         _dSize=`du -k ${_lf} | tr -d '\t' | cut -d '/' -f 1`
+         if [ $(is_num "$_dSize") ] ; then
+            if [ ${_fSize} -lt ${_dSize} ] ; then _dSize="$_fSize" ; fi
+	    _kbs=`expr ${_dSize} \/ $_time`
+	    echo "SIZE: ${_fSize} DOWNLOADED: ${_dSize} SPEED: ${_kbs} KB/s"
+  	 fi
+      fi
+
+      # Make sure download isn't finished
+      ps -p $FETCH_PID >/dev/null 2>/dev/null
+      if [ "$?" != "0" ] ; then break ; fi
+      sleep 2
+      _time=`expr $_time + 2`
+   done
+
+   _err="`cat ${_eFile}`"
+   rm ${_eFile} 2>/dev/null
+   if [ "$_err" = "0" ]; then echo "FETCHDONE" ; fi
+   unset FETCH_PID
+   return $_err
+
+}
+
 # Function to download a file from remote using fetch
 # Arg1 = Remote File URL
 # Arg2 = Where to save file
@@ -102,6 +202,7 @@
 		done
 
 		_err="`cat ${_eFile}`"
+                rm ${_eFile} 2>/dev/null
 		if [ "$_err" = "0" ]; then echo "FETCHDONE" ; fi
 		unset FETCH_PID
 	fi

Modified: pcbsd/current/src-sh/pbi-manager/pbi-manager
===================================================================
--- pcbsd/current/src-sh/pbi-manager/pbi-manager	2013-03-13 17:51:24 UTC (rev 21847)
+++ pcbsd/current/src-sh/pbi-manager/pbi-manager	2013-03-14 03:07:55 UTC (rev 21848)
@@ -5480,18 +5480,22 @@
 	for _cMirror in $_mirrorList
 	do
 		if [ "$_cMirror" = "PCBSDCDN" ] ; then
-			get_pcbsd_mirror
-			_furl="${VAL}${_uFile}"
+			get_file_from_mirrors "${_uFile}" "$_dl_loc"
+			if [ "$?" != "0" ]  ; then 
+				rm "${_dl_loc}" >/dev/null 2>/dev/null
+				echo "Download Failed: ${_furl}"
+				continue
+			fi
 		else
 			_furl="`echo $_cMirror | sed 's/\/*$//'`${_uFile}"
-		fi
 
-		echo "Downloading ${_furl}"
-		pbi_get_file "$_furl" "$_dl_loc"
-		if [ "$?" != "0" ]  ; then 
-			rm "${_dl_loc}" >/dev/null 2>/dev/null
-			echo "Download Failed: ${_furl}"
-			continue
+			echo "Downloading ${_furl}"
+			pbi_get_file "$_furl" "$_dl_loc"
+			if [ "$?" != "0" ]  ; then 
+				rm "${_dl_loc}" >/dev/null 2>/dev/null
+				echo "Download Failed: ${_furl}"
+				continue
+			fi
 		fi
 		_upcsum=`sha256 -q "$_dl_loc"`
 		if [ "$_CKSUM" != "OFF" -a "$_upcsum" != "$PBI_UPCSUM" ] ; then
@@ -6878,49 +6882,6 @@
 
 }
 
-# Get the default PC-BSD mirror to use
-get_pcbsd_mirror() {
-
-  # Check if we already looked up a mirror we can keep using
-  if [ -n "$CACHED_PCBSD_MIRROR" ] ; then
-     VAL="$CACHED_PCBSD_MIRROR"
-     export VAL
-     return
-  fi
-
-  # Set the mirror URL
-  VAL="`cat ${PCBSD_ETCCONF} 2>/dev/null | grep 'PCBSD_MIRROR: ' | sed 's|PCBSD_MIRROR: ||g'`"
-  if [ -n "$VAL" ] ; then
-     echo "Using mirror: $VAL"
-     CACHED_PCBSD_MIRROR="$VAL"
-     export VAL CACHED_PCBSD_MIRROR
-     return
-  fi
-
-  echo "Getting regional mirror..."
-  . /etc/profile
-
-  # No URL? Lets get one from the master server
-  local mFile="${HOME}/.mirrorUrl.$$"
-  touch $mFile
-  fetch -o $mFile http://getmirror.pcbsd.org >/dev/null 2>/dev/null
-  VAL="`cat $mFile | grep 'URL: ' | sed 's|URL: ||g'`"
-  rm $mFile
-  if [ -n "$VAL" ] ; then
-     echo "Using mirror: $VAL"
-     CACHED_PCBSD_MIRROR="$VAL"
-     export VAL CACHED_PCBSD_MIRROR
-     return
-  fi
-
-  # Still no mirror? Lets try the PC-BSD FTP server...
-  VAL="ftp://ftp.pcbsd.org/pub/mirror"
-  CACHED_PCBSD_MIRROR="$VAL"
-  export VAL CACHED_PCBSD_MIRROR
-  echo "Using mirror: $VAL"
-  return 
-}
-
 # No chroot environment tar file exists yet, lets build or extract
 mk_chroot_file() {
 
@@ -6929,10 +6890,6 @@
 
 	  cd "$PBI_APPDIR"
 
-	  # Set the mirror URL
-	  get_pcbsd_mirror
-          MIRRORURL="$VAL"
-
   	  # Get the system version we are checking for updates to
   	  SYSVER="`uname -r`" ; export SYSVER
 
@@ -6944,8 +6901,8 @@
           echo "Fetching chroot environment. This may take a while..."
           for i in $dFiles
           do
-            echo "Downloading ${MIRRORURL}/${SYSVER}/${ARCH}/dist/${i} ..."
-            fetch -o ${i} "${MIRRORURL}/${SYSVER}/${ARCH}/dist/${i}"
+            echo "Downloading /${SYSVER}/${ARCH}/dist/${i} ..."
+            get_file_from_mirrors "/${SYSVER}/${ARCH}/dist/${i}" "${i}"
             [ $? -ne 0 ] && exit_err "Error while downloading the freebsd world."
           done
 
@@ -7431,6 +7388,9 @@
   return 1
 }
 
+# Source our shared PCBSD functions
+. /usr/local/share/pcbsd/scripts/functions.sh
+
 # Main program operation
 ##############################################################
 init_vars

Modified: pcbsd/current/src-sh/pc-thinclient/pc-thinclient
===================================================================
--- pcbsd/current/src-sh/pc-thinclient/pc-thinclient	2013-03-13 17:51:24 UTC (rev 21847)
+++ pcbsd/current/src-sh/pc-thinclient/pc-thinclient	2013-03-14 03:07:55 UTC (rev 21848)
@@ -266,7 +266,7 @@
   do
     echo "Downloading ${MIRRORURL}/${SYSVER}/${ARCH}/dist/${i} ..."
     
-    get_file "${MIRRORURL}/${SYSVER}/${ARCH}/dist/${i}" "$i" 3
+    get_file_from_mirrors "/${SYSVER}/${ARCH}/dist/${i}" "$i"
     [ $? -ne 0 ] && exit_err "Error while downloading the freebsd world."
   done
 

Modified: pcbsd/current/src-sh/pc-updatemanager/pc-updatemanager
===================================================================
--- pcbsd/current/src-sh/pc-updatemanager/pc-updatemanager	2013-03-13 17:51:24 UTC (rev 21847)
+++ pcbsd/current/src-sh/pc-updatemanager/pc-updatemanager	2013-03-14 03:07:55 UTC (rev 21848)
@@ -315,7 +315,7 @@
   echo "DOWNLOADING: ${saIdent}" >${TRIGGERFILE}
 
   # Get the file
-  get_file "${MIRRORURL}/${saUrl}" "${DOWNLOADDIR}/${FILENAME}" 1
+  get_file_from_mirrors "/${saUrl}" "${DOWNLOADDIR}/${FILENAME}"
   if [ $? -ne 0 ] ; then
      rm ${PATCHTMPFILE}
      echo "FAILED: ${saIdent}" >${TRIGGERFILE}
@@ -471,9 +471,6 @@
     exit_err "No /usr/local/etc/pkg.conf!"
   fi
 
-  get_mirror 
-  local myMirror="$VAL"
-
   # Tickle pkg update first
   pkg-static update
 
@@ -513,9 +510,9 @@
 
   for i in $pkgList
   do
-    get_file "${myMirror}/${pkgUrl}/All/${i}" "${PKG_CACHEDIR}/All/${i}"
+    get_file_from_mirrors "/${pkgUrl}/All/${i}" "${PKG_CACHEDIR}/All/${i}"
     if [ $? -ne 0 ] ; then
-      exit_err "Failed downloading: ${myMirror}/${pkgUrl}/All/${i}"
+      exit_err "Failed downloading: /${pkgUrl}/All/${i}"
     fi
   done
 }

Modified: pcbsd/current/src-sh/port-files/Makefile
===================================================================
--- pcbsd/current/src-sh/port-files/Makefile	2013-03-13 17:51:24 UTC (rev 21847)
+++ pcbsd/current/src-sh/port-files/Makefile	2013-03-14 03:07:55 UTC (rev 21848)
@@ -17,6 +17,7 @@
 		svn:${PORTSDIR}/devel/subversion \
 		mksquashfs:${PORTSDIR}/sysutils/squashfs-tools \
 		debootstrap:${PORTSDIR}/sysutils/debootstrap \
+		aria2c:${PORTSDIR}/www/aria2 \
 		jailme:${PORTSDIR}/sysutils/jailme
 
 WRKSRC=		${WRKDIR}/src-sh

Modified: pcbsd/current/src-sh/port-files/pkg-plist
===================================================================
--- pcbsd/current/src-sh/port-files/pkg-plist	2013-03-13 17:51:24 UTC (rev 21847)
+++ pcbsd/current/src-sh/port-files/pkg-plist	2013-03-14 03:07:55 UTC (rev 21848)
@@ -49,6 +49,7 @@
 share/warden/scripts/backend/zfscronsnap.sh
 share/warden/linux-installs/debian-6-squeeze
 share/warden/linux-installs/gentoo-stage3-i486
+share/pcbsd/conf/pcbsd-mirrors
 share/pcbsd/conf/port-excludes
 share/pcbsd/conf/prune-port-files
 share/pcbsd/conf/desktop-excludes

Modified: pcbsd/current/src-sh/warden/scripts/backend/functions.sh
===================================================================
--- pcbsd/current/src-sh/warden/scripts/backend/functions.sh	2013-03-13 17:51:24 UTC (rev 21847)
+++ pcbsd/current/src-sh/warden/scripts/backend/functions.sh	2013-03-14 03:07:55 UTC (rev 21848)
@@ -70,7 +70,7 @@
   do
     echo "Downloading ${MIRRORURL}/${SYSVER}/${ARCH}/dist/${i} ..."
 
-    get_file "${MIRRORURL}/${SYSVER}/${ARCH}/dist/${i}" "$i" 3
+    get_file_from_mirrors "/${SYSVER}/${ARCH}/dist/${i}" "$i"
     [ $? -ne 0 ] && printerror "Error while downloading the freebsd world."
   done
 



More information about the Commits mailing list