Re: Batch Download Locks Server
Florian Maul <[email protected]> Wed, 6 Aug 2003 14:52:27 +0200
| Newsgroups | gmane.comp.audio.netjuke.user |
|---|---|
| Message-ID | <[email protected]> |
Hello John and CVS/rc1-users,
> Whenever users select multiple files for batch download, the CPU on
> the server goes straight up to 99-100% and will stay there
> indefinitely (I have let it run for more than hour for 3 files)
> until killed. Has anyone seen this before?
Thanks for your pointer, John. Blake implemented an internal php zip
function to make batch downloading system independant. I just had
a closer look and realized that the library he used was horribly
inefficient with large files
- Compression of the whole archive in memory (a lot of ram and high
php-mem-limit required)
- Very inefficient handling of the whole archive data required
memory more than three times the archive-size!
- A long waiting (compile) time before the download started.
- compression is not necessary for ogg/mp3 files
I fixed all of these issues now in CVS and hope it'll work more
smoothly for all of you. I'd appreciate any comments by the CVS-users
out there if this code is releaseable for 1.0. I also attached the
files batch.php and lib/zipfile/zipfile.php for the 1.0rc1 users who
like to test my changes.
requirements for (internal) batch downloading:
- php mem-limit at least 2 times the biggest file you want package
(the common default 16M could be to low in some cases)
- php time limit has to be high enough (netjuke setting BATCH_TIME_LIMIT)
- php has to have zlib support (it usually has)
Florian <[email protected]>
batch.php
(application/octet-stream, 2.8 KB)
<?php
// Try to turn off the gz handler
if (ini_get("safe_mode") == 0) {
ini_set ("zlib.output_compression", "0");
}
// defines if this script requires to be logged in
define( "PRIVATE", false );
# CALL COMMON LIBRARIES
require_once('./lib/inc-common.php');
require_once(FS_PATH."/etc/locale/".LANG_PACK."/inc-batch.php");
require_once(FS_PATH."/lib/zipfile/zipfile.php");
// Don't try to subvert me!
if (BATCH_DOWNLOAD == 'f') {
header ("Location: index.php\n\n");
}
set_time_limit (BATCH_TIME_LIMIT);
if (strtolower($_REQUEST['do']) == 'download') {
bundleMe ($_REQUEST['val']);
} else {
header ("Location: index.php\n\n");
}
function bundleMe ($val) {
GLOBAL $dbconn, $NETJUKE_SESSION_VARS;
$total_size = 0;
$files = "";
$zipfile = new zipfile();
if ($val != '') {
ignore_user_abort (false);
header("Content-type: application/octet-stream");
header("Content-Disposition: attachment; filename=netjuke-".substr(time(),-7).".zip" );
$id = split(",",$val);
$trans = get_html_translation_table (HTML_ENTITIES);
$trans = array_flip ($trans);
foreach ($id as $this_id) {
$sql = "SELECT tr.location, tr.time, tr.name, ar.name, al.name "
. " from netjuke_tracks tr, netjuke_artists ar, netjuke_albums al "
. " where tr.id = $this_id and ar.id = tr.ar_id and al.id = tr.al_id ";
$dbrs = $dbconn->Execute($sql);
$tr_location = rawurldecode(separatorCleanup($dbrs->fields[0]));
$tr_time = floor($dbrs->fields[1]);
$tr_name = strtr(format_for_display($dbrs->fields[2]),$trans);
$ar_name = strtr(format_for_display($dbrs->fields[3]),$trans);
$al_name = strtr(format_for_display($dbrs->fields[4]),$trans);
# Append web path value if location still doesn't contain ://
# Copes with the fact that the streaming server could be a relative path (eg: music/)
# instead of a protocol+hostname combination (http://streaming.host.dom)
#if (!strstr($tr_location,"://")) $tr_location = WEB_PATH."/".$tr_location;
// adding file to zip archiv, compression level 0
$zipfile->add_file(MUSIC_DIR.'/'.$tr_location, basename($tr_location), 0);
$zipfile->echo_data();
// update the track's download count.
// Is normally done accurately in the dispatcher, but it is now optional...
$dbconn->Execute( "update netjuke_tracks set dl_cnt = dl_cnt + 1 where id = ".$this_id." " );
$dbrs->Close();
if (BATCH_SIZE_LIMIT != '0') {
$total_size += filesize (MUSIC_DIR . '/' . rawurldecode($tr_location));
if (($total_size / (1024 * 1024)) >= intval (BATCH_SIZE_LIMIT)) {
break;
}
} // if BATCH_SIZE_LIMIT
} // foreach $id
$zipfile->echo_footer();
}
// else $val == ""
else {
alert (PLAYER_NOID);
}
exit;
} // function bundleMe()
?>
zipfile.php
(application/octet-stream, 8.6 KB)
<?php /* Zip file creation class makes zip files on the fly... altered use in netjuke for more efficient building of large zip files use the functions add_dir() and add_file() to build the zip file; see example code below by Eric Mueller http://www.themepark.com v1.1 9-20-01 - added comments to example v1.0 2-5-01 initial version with: - class appearance - add_file() and file() methods - gzcompress() output hacking by Denis O.Philippov, [email protected], http://www.atlant.ru */ // official ZIP file format: http://www.pkware.com/appnote.txt class zipfile { var $datasec = array(); // array to store compressed data var $datasize = array(); // sizes of datasec[] var $ctrl_dir = array(); // central directory var $eof_ctrl_dir = "\x50\x4b\x05\x06\x00\x00\x00\x00"; //end of Central directory record var $old_offset = 0; function add_dir($name) // adds "directory" to archive - do this before putting any files in directory! // $name - name of directory... like this: "path/" // ...then you can add files using add_file with names like "path/file.txt" { $name = str_replace("\\", "/", $name); $fr = "\x50\x4b\x03\x04"; $fr .= "\x0a\x00"; // ver needed to extract $fr .= "\x00\x00"; // gen purpose bit flag $fr .= "\x00\x00"; // compression method $fr .= "\x00\x00\x00\x00"; // last mod time and date $fr .= pack("V",0); // crc32 $fr .= pack("V",0); //compressed filesize $fr .= pack("V",0); //uncompressed filesize $fr .= pack("v", strlen($name) ); //length of pathname $fr .= pack("v", 0 ); //extra field length $fr .= $name; // end of "local file header" segment // no "file data" segment for path // "data descriptor" segment (optional but necessary if archive is not served as file) $fr .= pack("V",$crc); //crc32 $fr .= pack("V",$c_len); //compressed filesize $fr .= pack("V",$unc_len); //uncompressed filesize // add this entry to array $this -> datasec[] = $fr; $this -> datasize[] = strlen($this->datasec[count($this->datasec)-1]); $new_offset = 0; for ($i=0; $i < count($this->datasize); $i++) $new_offset += $this->datasize[$i]; // ext. file attributes mirrors MS-DOS directory attr byte, detailed // at http://support.microsoft.com/support/kb/articles/Q125/0/19.asp // now add to central record $cdrec = "\x50\x4b\x01\x02"; $cdrec .="\x00\x00"; // version made by $cdrec .="\x0a\x00"; // version needed to extract $cdrec .="\x00\x00"; // gen purpose bit flag $cdrec .="\x00\x00"; // compression method $cdrec .="\x00\x00\x00\x00"; // last mod time & date $cdrec .= pack("V",0); // crc32 $cdrec .= pack("V",0); //compressed filesize $cdrec .= pack("V",0); //uncompressed filesize $cdrec .= pack("v", strlen($name) ); //length of filename $cdrec .= pack("v", 0 ); //extra field length $cdrec .= pack("v", 0 ); //file comment length $cdrec .= pack("v", 0 ); //disk number start $cdrec .= pack("v", 0 ); //internal file attributes $ext = "\x00\x00\x10\x00"; $ext = "\xff\xff\xff\xff"; $cdrec .= pack("V", 16 ); //external file attributes - 'directory' bit set $cdrec .= pack("V", $this -> old_offset ); //relative offset of local header $this -> old_offset = $new_offset; $cdrec .= $name; // optional extra field, file comment goes here // save to array $this -> ctrl_dir[] = $cdrec; } function add_file($datafilename, $name, $compression = 5) // adds "file" to archive // $data - file contents // $name - name of file in archive. Add path if your want { $handle = fopen ($datafilename, "rb"); $data = fread ($handle, filesize ($datafilename)); fclose ($handle); $name = str_replace("\\", "/", $name); //$name = str_replace("\\", "\\\\", $name); $fr = "\x50\x4b\x03\x04"; $fr .= "\x14\x00"; // ver needed to extract $fr .= "\x00\x00"; // gen purpose bit flag $fr .= "\x08\x00"; // compression method $fr .= "\x00\x00\x00\x00"; // last mod time and date $unc_len = strlen($data); $crc = crc32($data); $zdata = gzcompress($data, $compression); unset($data); // fix crc bug: remove first 2 and last 4 bytes $zdata = substr($zdata, 2, strlen($zdata)-6); $c_len = strlen($zdata); $fr .= pack("V",$crc); // crc32 $fr .= pack("V",$c_len); //compressed filesize $fr .= pack("V",$unc_len); //uncompressed filesize $fr .= pack("v", strlen($name) ); //length of filename $fr .= pack("v", 0 ); //extra field length $fr .= $name; // end of "local file header" segment // "data descriptor" segment (optional but necessary if archive is not served as file) // add this entry to array $this -> datasec[] = $fr.$zdata .pack("V",$crc) //crc32 .pack("V",$c_len) //compressed filesize .pack("V",$unc_len); //uncompressed filesize unset($zdata); $this->datasize[] = strlen($this->datasec[count($this->datasec)-1]); $new_offset = 0; for ($i=0; $i < count($this->datasize); $i++) $new_offset += $this->datasize[$i]; // now add to central directory record $cdrec = "\x50\x4b\x01\x02"; $cdrec .="\x00\x00"; // version made by $cdrec .="\x14\x00"; // version needed to extract $cdrec .="\x00\x00"; // gen purpose bit flag $cdrec .="\x08\x00"; // compression method $cdrec .="\x00\x00\x00\x00"; // last mod time & date $cdrec .= pack("V",$crc); // crc32 $cdrec .= pack("V",$c_len); //compressed filesize $cdrec .= pack("V",$unc_len); //uncompressed filesize $cdrec .= pack("v", strlen($name) ); //length of filename $cdrec .= pack("v", 0 ); //extra field length $cdrec .= pack("v", 0 ); //file comment length $cdrec .= pack("v", 0 ); //disk number start $cdrec .= pack("v", 0 ); //internal file attributes $cdrec .= pack("V", 32 ); //external file attributes - 'archive' bit set $cdrec .= pack("V", $this -> old_offset ); //relative offset of local header // echo "old offset is ".$this->old_offset.", new offset is $new_offset<br>"; $this -> old_offset = $new_offset; $cdrec .= $name; // optional extra field, file comment goes here // save to central directory $this -> ctrl_dir[] = $cdrec; } function file() { // dump out file $data = implode("", $this -> datasec); $ctrldir = implode("", $this -> ctrl_dir); return $data. $ctrldir. $this -> eof_ctrl_dir. pack("v", sizeof($this -> ctrl_dir)). // total # of entries "on this disk" pack("v", sizeof($this -> ctrl_dir)). // total # of entries overall pack("V", strlen($ctrldir)). // size of central dir pack("V", strlen($data)). // offset to start of central dir "\x00\x00"; // .zip file comment length } // print all pending data sections to stdout // optimized for large files // function echo_data() { for ($i=0; $i < count($this->datasec); $i++) { if ($this->datasec[$i] != null) { echo $this->datasec[$i]; $this->datasec[$i] = null; } } } // print zipfile-footer to stdout // optimized for large files // function echo_footer() { $len_ctrl_dir = 0; $len_datasec = 0; for ($i = 0; $i < count($this->datasize); $i++) { $len_datasec += $this->datasize[$i]; } for ($i = 0; $i < count($this->ctrl_dir); $i++) { echo $this->ctrl_dir[$i]; $len_ctrl_dir += strlen($this->ctrl_dir[$i]); } echo $this -> eof_ctrl_dir; echo pack("v", sizeof($this->ctrl_dir)). // total # of entries "on this disk" pack("v", sizeof($this->ctrl_dir)). // total # of entries overall pack("V", $len_ctrl_dir). // size of central dir pack("V", $len_datasec). // offset to start of central dir "\x00\x00"; // .zip file comment length } } ?>