Re: Batch Download Locks Server

Florian Maul <[email protected]> Wed, 6 Aug 2003 14:52:27 +0200
Newsgroups gmane.comp.audio.netjuke.user
Message-ID <[email protected]>
Hello John and CVS/rc1-users,

> Whenever users select multiple files for batch download, the CPU on
> the server goes straight up to 99-100% and will stay there
> indefinitely (I have let it run for more than hour for 3 files)
> until killed. Has anyone seen this before?
Thanks for your pointer, John. Blake implemented an internal php zip
function to make batch downloading system independant. I just had
a closer look and realized that the library he used was horribly
inefficient with large files

  - Compression of the whole archive in memory (a lot of ram and high
    php-mem-limit required)
  - Very inefficient handling of the whole archive data required
    memory more than three times the archive-size!
  - A long waiting (compile) time before the download started.
  - compression is not necessary for ogg/mp3 files

I fixed all of these issues now in CVS and hope it'll work more
smoothly for all of you. I'd appreciate any comments by the CVS-users
out there if this code is releaseable for 1.0. I also attached the
files batch.php and lib/zipfile/zipfile.php for the 1.0rc1 users who
like to test my changes.

requirements for (internal) batch downloading:
  - php mem-limit at least 2 times the biggest file you want package
    (the common default 16M could be to low in some cases)
  - php time limit has to be high enough (netjuke setting BATCH_TIME_LIMIT)
  - php has to have zlib support (it usually has)

Florian <[email protected]>
batch.php (application/octet-stream, 2.8 KB)
<?php

// Try to turn off the gz handler
if (ini_get("safe_mode") == 0) {
  ini_set ("zlib.output_compression", "0");
}

// defines if this script requires to be logged in
define( "PRIVATE", false );

# CALL COMMON LIBRARIES

require_once('./lib/inc-common.php');
require_once(FS_PATH."/etc/locale/".LANG_PACK."/inc-batch.php");
require_once(FS_PATH."/lib/zipfile/zipfile.php");

// Don't try to subvert me!
if (BATCH_DOWNLOAD == 'f') {

  header ("Location: index.php\n\n");

}

set_time_limit (BATCH_TIME_LIMIT);

if (strtolower($_REQUEST['do']) == 'download') {
	bundleMe ($_REQUEST['val']);
} else {
	header ("Location: index.php\n\n");
}

function bundleMe ($val) {

  GLOBAL $dbconn, $NETJUKE_SESSION_VARS;

  $total_size = 0;
  $files = "";
	
  $zipfile = new zipfile(); 

  if ($val != '') {

    ignore_user_abort (false);
    header("Content-type: application/octet-stream");
    header("Content-Disposition: attachment; filename=netjuke-".substr(time(),-7).".zip" );

    $id = split(",",$val);

    $trans = get_html_translation_table (HTML_ENTITIES);
    $trans = array_flip ($trans);

    foreach ($id as $this_id) {

      $sql = "SELECT tr.location, tr.time, tr.name, ar.name, al.name  "
        . " from netjuke_tracks tr, netjuke_artists ar, netjuke_albums al "
        . " where tr.id = $this_id and ar.id = tr.ar_id and al.id = tr.al_id ";

      $dbrs = $dbconn->Execute($sql);

      $tr_location  = rawurldecode(separatorCleanup($dbrs->fields[0]));
      $tr_time      = floor($dbrs->fields[1]);
      $tr_name      = strtr(format_for_display($dbrs->fields[2]),$trans);
      $ar_name      = strtr(format_for_display($dbrs->fields[3]),$trans);
      $al_name      = strtr(format_for_display($dbrs->fields[4]),$trans);

      # Append web path value if location still doesn't contain ://
      # Copes with the fact that the streaming server could be a relative path (eg: music/)
      # instead of a protocol+hostname combination (http://streaming.host.dom)
      #if (!strstr($tr_location,"://")) $tr_location  = WEB_PATH."/".$tr_location;

      // adding file to zip archiv, compression level 0
      $zipfile->add_file(MUSIC_DIR.'/'.$tr_location, basename($tr_location), 0);
      $zipfile->echo_data();

      // update the track's download count.
      // Is normally done accurately in the dispatcher, but it is now optional...
      $dbconn->Execute( "update netjuke_tracks set dl_cnt = dl_cnt + 1 where id = ".$this_id." " );

      $dbrs->Close();

      if (BATCH_SIZE_LIMIT != '0') {

        $total_size += filesize (MUSIC_DIR . '/' . rawurldecode($tr_location));
        if (($total_size / (1024 * 1024)) >= intval (BATCH_SIZE_LIMIT)) {

            break;

        }

      } // if BATCH_SIZE_LIMIT

    } // foreach $id

    $zipfile->echo_footer();
  } 

  // else $val == ""
  else {
    alert (PLAYER_NOID);
  }

  exit;

} // function bundleMe()

?>
zipfile.php (application/octet-stream, 8.6 KB)
<?php

/*

Zip file creation class
makes zip files on the fly...
altered use in netjuke for more efficient building of large zip files

use the functions add_dir() and add_file() to build the zip file;
see example code below

by Eric Mueller
http://www.themepark.com

v1.1 9-20-01
  - added comments to example

v1.0 2-5-01

initial version with:
  - class appearance
  - add_file() and file() methods
  - gzcompress() output hacking
by Denis O.Philippov, [email protected], http://www.atlant.ru

*/  

// official ZIP file format: http://www.pkware.com/appnote.txt

class zipfile   
{   

    var $datasec = array();  // array to store compressed data
    var $datasize = array(); // sizes of datasec[]
    var $ctrl_dir = array(); // central directory    
    var $eof_ctrl_dir = "\x50\x4b\x05\x06\x00\x00\x00\x00"; //end of Central directory record
    var $old_offset = 0;

    function add_dir($name)    

    // adds "directory" to archive - do this before putting any files in directory!
    // $name - name of directory... like this: "path/"
    // ...then you can add files using add_file with names like "path/file.txt"
    {   
        $name = str_replace("\\", "/", $name);   

        $fr = "\x50\x4b\x03\x04";  
        $fr .= "\x0a\x00";    // ver needed to extract
        $fr .= "\x00\x00";    // gen purpose bit flag
        $fr .= "\x00\x00";    // compression method
        $fr .= "\x00\x00\x00\x00"; // last mod time and date

        $fr .= pack("V",0); // crc32
        $fr .= pack("V",0); //compressed filesize
        $fr .= pack("V",0); //uncompressed filesize
        $fr .= pack("v", strlen($name) ); //length of pathname
        $fr .= pack("v", 0 ); //extra field length
        $fr .= $name;   
        // end of "local file header" segment

        // no "file data" segment for path

        // "data descriptor" segment (optional but necessary if archive is not served as file)
        $fr .= pack("V",$crc); //crc32
        $fr .= pack("V",$c_len); //compressed filesize
        $fr .= pack("V",$unc_len); //uncompressed filesize

        // add this entry to array
        $this -> datasec[] = $fr;  
        $this -> datasize[] = strlen($this->datasec[count($this->datasec)-1]);

        $new_offset = 0;
        for ($i=0; $i < count($this->datasize); $i++) 
          $new_offset += $this->datasize[$i];  

        // ext. file attributes mirrors MS-DOS directory attr byte, detailed
        // at http://support.microsoft.com/support/kb/articles/Q125/0/19.asp

        // now add to central record
        $cdrec = "\x50\x4b\x01\x02";  
        $cdrec .="\x00\x00";    // version made by
        $cdrec .="\x0a\x00";    // version needed to extract
        $cdrec .="\x00\x00";    // gen purpose bit flag
        $cdrec .="\x00\x00";    // compression method
        $cdrec .="\x00\x00\x00\x00"; // last mod time & date
        $cdrec .= pack("V",0); // crc32
        $cdrec .= pack("V",0); //compressed filesize
        $cdrec .= pack("V",0); //uncompressed filesize
        $cdrec .= pack("v", strlen($name) ); //length of filename
        $cdrec .= pack("v", 0 ); //extra field length    
        $cdrec .= pack("v", 0 ); //file comment length
        $cdrec .= pack("v", 0 ); //disk number start
        $cdrec .= pack("v", 0 ); //internal file attributes
        $ext = "\x00\x00\x10\x00";  
        $ext = "\xff\xff\xff\xff";   
        $cdrec .= pack("V", 16 ); //external file attributes  - 'directory' bit set

        $cdrec .= pack("V", $this -> old_offset ); //relative offset of local header
        $this -> old_offset = $new_offset;  

        $cdrec .= $name;   
        // optional extra field, file comment goes here
        // save to array
        $this -> ctrl_dir[] = $cdrec;   

          
    }  


    function add_file($datafilename, $name, $compression = 5)    

    // adds "file" to archive    
    // $data - file contents
    // $name - name of file in archive. Add path if your want

    {   
        $handle = fopen ($datafilename, "rb");
        $data = fread ($handle, filesize ($datafilename));
        fclose ($handle);

        $name = str_replace("\\", "/", $name);   
        //$name = str_replace("\\", "\\\\", $name);

        $fr = "\x50\x4b\x03\x04";  
        $fr .= "\x14\x00";    // ver needed to extract
        $fr .= "\x00\x00";    // gen purpose bit flag
        $fr .= "\x08\x00";    // compression method
        $fr .= "\x00\x00\x00\x00"; // last mod time and date

        $unc_len = strlen($data);   
        $crc = crc32($data);   
        $zdata = gzcompress($data, $compression);   
        unset($data);

        // fix crc bug: remove first 2 and last 4 bytes
        $zdata = substr($zdata, 2, strlen($zdata)-6);

        $c_len = strlen($zdata);   
        $fr .= pack("V",$crc); // crc32
        $fr .= pack("V",$c_len); //compressed filesize
        $fr .= pack("V",$unc_len); //uncompressed filesize
        $fr .= pack("v", strlen($name) ); //length of filename
        $fr .= pack("v", 0 ); //extra field length
        $fr .= $name;   
        // end of "local file header" segment

        // "data descriptor" segment (optional but necessary if archive is not served as file)
        // add this entry to array
        $this -> datasec[] = $fr.$zdata  
          .pack("V",$crc)      //crc32
          .pack("V",$c_len)    //compressed filesize
          .pack("V",$unc_len); //uncompressed filesize
        unset($zdata);
        $this->datasize[] = strlen($this->datasec[count($this->datasec)-1]);

        $new_offset = 0;
        for ($i=0; $i < count($this->datasize); $i++) 
          $new_offset += $this->datasize[$i];  

        // now add to central directory record
        $cdrec = "\x50\x4b\x01\x02";  
        $cdrec .="\x00\x00";    // version made by
        $cdrec .="\x14\x00";    // version needed to extract
        $cdrec .="\x00\x00";    // gen purpose bit flag
        $cdrec .="\x08\x00";    // compression method
        $cdrec .="\x00\x00\x00\x00"; // last mod time & date
        $cdrec .= pack("V",$crc); // crc32
        $cdrec .= pack("V",$c_len); //compressed filesize
        $cdrec .= pack("V",$unc_len); //uncompressed filesize
        $cdrec .= pack("v", strlen($name) ); //length of filename
        $cdrec .= pack("v", 0 ); //extra field length    
        $cdrec .= pack("v", 0 ); //file comment length
        $cdrec .= pack("v", 0 ); //disk number start
        $cdrec .= pack("v", 0 ); //internal file attributes
        $cdrec .= pack("V", 32 ); //external file attributes - 'archive' bit set

        $cdrec .= pack("V", $this -> old_offset ); //relative offset of local header
//        echo "old offset is ".$this->old_offset.", new offset is $new_offset<br>";
        $this -> old_offset = $new_offset;  

        $cdrec .= $name;   
        // optional extra field, file comment goes here
        // save to central directory
        $this -> ctrl_dir[] = $cdrec;
    }  

    function file() { // dump out file    
        $data = implode("", $this -> datasec);   
        $ctrldir = implode("", $this -> ctrl_dir);   

        return    
            $data.   
            $ctrldir.   
            $this -> eof_ctrl_dir.   
            pack("v", sizeof($this -> ctrl_dir)).     // total # of entries "on this disk"
            pack("v", sizeof($this -> ctrl_dir)).     // total # of entries overall
            pack("V", strlen($ctrldir)).             // size of central dir
            pack("V", strlen($data)).                 // offset to start of central dir
            "\x00\x00";                             // .zip file comment length
    }  

    // print all pending data sections to stdout
    // optimized for large files
    //
    function echo_data() {
      for ($i=0; $i < count($this->datasec); $i++) {
        if ($this->datasec[$i] != null) {
          echo $this->datasec[$i];
          $this->datasec[$i] = null;
        }
      }
    }    

    // print zipfile-footer to stdout
    // optimized for large files
    //
    function echo_footer() {    
        $len_ctrl_dir = 0;
        $len_datasec = 0;

        for ($i = 0; $i < count($this->datasize); $i++) {
          $len_datasec += $this->datasize[$i];   
        }

        for ($i = 0; $i < count($this->ctrl_dir); $i++) {
          echo $this->ctrl_dir[$i];
          $len_ctrl_dir += strlen($this->ctrl_dir[$i]);   
        }

        echo $this -> eof_ctrl_dir;

        echo pack("v", sizeof($this->ctrl_dir)).    // total # of entries "on this disk"
          pack("v", sizeof($this->ctrl_dir)).       // total # of entries overall
          pack("V", $len_ctrl_dir).         // size of central dir
          pack("V", $len_datasec).          // offset to start of central dir
          "\x00\x00";                               // .zip file comment length
    }  
}   

?>