aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorplegall <plg@piwigo.org>2009-03-12 23:09:22 +0000
committerplegall <plg@piwigo.org>2009-03-12 23:09:22 +0000
commit497e753bcf4208e1adc457dbc0f406b923594343 (patch)
tree1c1db805edda1f0e928c64651d00dbd8c3618738
parentb70496545c8b5af966cc1fb99d35dda92f8f3a81 (diff)
bug 941 fixed: to be able to upload heavy photo, chunk the files, send parts
one by one, and then pwg.images.add merge chunks together. Now big uploads works and you can even have a fine progress bar on client side. git-svn-id: http://piwigo.org/svn/branches/2.0@3192 68402e56-0260-453c-a942-63ccdbb3a9ee
-rw-r--r--include/ws_functions.inc.php134
-rw-r--r--tools/piwigo_remote.pl100
-rw-r--r--ws.php16
3 files changed, 206 insertions, 44 deletions
diff --git a/include/ws_functions.inc.php b/include/ws_functions.inc.php
index a85fa2e4b..7c1d52580 100644
--- a/include/ws_functions.inc.php
+++ b/include/ws_functions.inc.php
@@ -879,6 +879,98 @@ UPDATE '.IMAGES_TABLE.'
return $affected_rows;
}
+function ws_images_add_chunk($params, &$service)
+{
+ // data
+ // original_sum
+ // type {thumb, file, high}
+ // position
+
+ if (!is_admin() || is_adviser() )
+ {
+ return new PwgError(401, 'Access denied');
+ }
+
+ $upload_dir = PHPWG_ROOT_PATH.'upload/buffer';
+
+ // create the upload directory tree if not exists
+ if (!is_dir($upload_dir)) {
+ umask(0000);
+ $recursive = true;
+ if (!@mkdir($upload_dir, 0777, $recursive))
+ {
+ return new PwgError(500, 'error during buffer directory creation');
+ }
+ }
+
+ if (!is_writable($upload_dir))
+ {
+ // last chance to make the directory writable
+ @chmod($upload_dir, 0777);
+
+ if (!is_writable($upload_dir))
+ {
+ return new PwgError(500, 'buffer directory has no write access');
+ }
+ }
+
+ secure_directory($upload_dir);
+
+ $filename = sprintf(
+ '%s-%s-%05u.block',
+ $params['original_sum'],
+ $params['type'],
+ $params['position']
+ );
+
+ $bytes_written = file_put_contents(
+ $upload_dir.'/'.$filename,
+ $params['data']
+ );
+
+ if (false === $bytes_written) {
+ return new PwgError(
+ 500,
+ 'an error has occured while writting chunk '.$params['position'].' for '.$params['type']
+ );
+ }
+}
+
+function merge_chunks($output_filepath, $original_sum, $type)
+{
+ ws_logfile('[merge_chunks] input parameter $output_filepath : '.$output_filepath);
+
+ $upload_dir = PHPWG_ROOT_PATH.'upload/buffer';
+ $pattern = '/'.$original_sum.'-'.$type.'/';
+ $chunks = array();
+
+ if ($handle = opendir($upload_dir))
+ {
+ while (false !== ($file = readdir($handle)))
+ {
+ if (preg_match($pattern, $file))
+ {
+ ws_logfile($file);
+ array_push($chunks, $upload_dir.'/'.$file);
+ }
+ }
+ closedir($handle);
+ }
+
+ sort($chunks);
+
+ $string = null;
+ foreach ($chunks as $chunk) {
+ $string.= file_get_contents($chunk);
+ unlink($chunk);
+ }
+ if (!file_put_contents($output_filepath, base64_decode($string)))
+ {
+ return new PwgError(500, 'error while merging chunks for '.$output_filepath);
+ }
+
+}
+
function ws_images_add($params, &$service)
{
global $conf;
@@ -956,13 +1048,8 @@ SELECT
$filename_wo_ext = $date_string.'-'.$random_string;
$file_path = $upload_dir.'/'.$filename_wo_ext.'.jpg';
- // dump the photo file
- $fh_file = fopen($file_path, 'w');
- if (!fwrite($fh_file, base64_decode($params['file_content'])))
- {
- return new PwgError(500, 'error while writing file');
- }
- fclose($fh_file);
+ // merge the photo file
+ merge_chunks($file_path, $params['original_sum'], 'file');
chmod($file_path, 0644);
// check dumped file md5sum against expected md5sum
@@ -1005,13 +1092,8 @@ SELECT
'jpg'
);
- // dump the thumbnail
- $fh_thumbnail = fopen($thumbnail_path, 'w');
- if (!fwrite($fh_thumbnail, base64_decode($params['thumbnail_content'])))
- {
- return new PwgError(500, 'error while writing thumbnail');
- }
- fclose($fh_thumbnail);
+ // merge the thumbnail
+ merge_chunks($thumbnail_path, $params['original_sum'], 'thumb');
chmod($thumbnail_path, 0644);
// check dumped thumbnail md5
@@ -1021,7 +1103,7 @@ SELECT
}
// high resolution
- if (isset($params['high_content']))
+ if (isset($params['high_sum']))
{
// high resolution directory is a subdirectory of the photo file, hard
// coded "pwg_high"
@@ -1055,13 +1137,8 @@ SELECT
'jpg'
);
- // dump the high resolution file
- $fh_high = fopen($high_path, 'w');
- if (!fwrite($fh_high, base64_decode($params['high_content'])))
- {
- return new PwgError(500, 'error while writing high');
- }
- fclose($fh_high);
+ // merge the high resolution file
+ merge_chunks($high_path, $params['original_sum'], 'high');
chmod($high_path, 0644);
// check dumped thumbnail md5
@@ -1104,7 +1181,7 @@ SELECT
}
}
- if (isset($params['high_content']))
+ if (isset($params['high_sum']))
{
$insert['has_high'] = 'true';
$insert['high_filesize'] = $high_filesize;
@@ -1646,4 +1723,15 @@ SELECT
update_category($cat_ids);
}
}
+
+function ws_logfile($string)
+{
+ return true;
+
+ file_put_contents(
+ '/tmp/piwigo_ws.log',
+ '['.date('c').'] '.$string."\n",
+ FILE_APPEND
+ );
+}
?>
diff --git a/tools/piwigo_remote.pl b/tools/piwigo_remote.pl
index 6cd558cd1..91142f25a 100644
--- a/tools/piwigo_remote.pl
+++ b/tools/piwigo_remote.pl
@@ -1,5 +1,17 @@
#!/usr/bin/perl
+####
+# Usage examples
+#
+# time perl piwigo_remote.pl \
+# --action=pwg.images.add \
+# --file=erwann_rocher-web.jpg \
+# --thumb=erwann_rocher-thumb.jpg \
+# --high=erwann_rocher-high.jpg \
+# --original=erwann_rocher-high.jpg \
+# --define categories=9 \
+# --chunk_size=200_000
+
use strict;
use warnings;
@@ -7,22 +19,24 @@ use JSON;
use LWP::UserAgent;
use Getopt::Long;
use Encode qw/is_utf8 decode/;
+use POSIX qw(ceil floor);
my %opt = ();
GetOptions(
\%opt,
- qw/action=s file=s thumbnail=s high=s original=s categories=s define=s%/
+ qw/action=s file=s thumbnail=s high=s original=s categories=s chunk_size=i define=s%/
);
our $ua = LWP::UserAgent->new;
$ua->cookie_jar({});
my %conf;
-$conf{base_url} = 'http://localhost/~pierrick/piwigo/2.0';
+$conf{base_url} = 'http://localhost/piwigo/2.0';
$conf{response_format} = 'json';
-$conf{username} = 'pierrick';
-$conf{password} = 'z0rglub';
+$conf{username} = 'plg';
+$conf{password} = 'plg';
$conf{limit} = 10;
+$conf{chunk_size} = defined $opt{chunk_size} ? $opt{chunk_size} : 500_000;
my $result = undef;
my $query = undef;
@@ -48,26 +62,32 @@ if ($opt{action} eq 'pwg.images.add') {
use Digest::MD5::File qw/file_md5_hex/;
use File::Slurp;
- my $original_sum = file_md5_hex($opt{original});
+ $form = {};
+ $form->{method} = 'pwg.images.add';
- my $file_content = encode_base64(read_file($opt{file}));
- my $file_sum = file_md5_hex($opt{file});
+ my $original_sum = file_md5_hex($opt{original});
+ $form->{original_sum} = $original_sum;
- my $thumbnail_content = encode_base64(read_file($opt{thumbnail}));
- my $thumbnail_sum = file_md5_hex($opt{thumbnail});
+ send_chunks(
+ filepath => $opt{file},
+ type => 'file',
+ original_sum => $original_sum,
+ );
+ $form->{file_sum} = file_md5_hex($opt{file});
- $form = {
- method => 'pwg.images.add',
+ send_chunks(
+ filepath => $opt{thumbnail},
+ type => 'thumb',
original_sum => $original_sum,
- file_sum => $file_sum,
- file_content => $file_content,
- thumbnail_sum => $thumbnail_sum,
- thumbnail_content => $thumbnail_content,
- categories => $opt{categories},
- };
+ );
+ $form->{thumbnail_sum} = file_md5_hex($opt{thumbnail});
if (defined $opt{high}) {
- $form->{high_content} = encode_base64(read_file($opt{high}));
+ send_chunks(
+ filepath => $opt{high},
+ type => 'high',
+ original_sum => $original_sum,
+ );
$form->{high_sum} = file_md5_hex($opt{high});
}
@@ -231,3 +251,47 @@ sub pwg_ws_get_query {
return $query;
}
+
+sub send_chunks {
+ my %params = @_;
+
+ my $content = encode_base64(read_file($params{filepath}));
+ my $content_length = length($content);
+ my $nb_chunks = ceil($content_length / $conf{chunk_size});
+
+ my $chunk_pos = 0;
+ my $chunk_id = 1;
+ while ($chunk_pos < $content_length) {
+ my $chunk = substr(
+ $content,
+ $chunk_pos,
+ $conf{chunk_size}
+ );
+ $chunk_pos += $conf{chunk_size};
+
+ my $response = $ua->post(
+ $conf{base_url}.'/ws.php?format=json',
+ {
+ method => 'pwg.images.addChunk',
+ data => $chunk,
+ original_sum => $params{original_sum},
+ position => $chunk_id,
+ type => $params{type},
+ }
+ );
+
+ printf(
+ 'chunk %05u of %05u for %s "%s"'."\n",
+ $chunk_id,
+ $nb_chunks,
+ $params{type},
+ $params{filepath}
+ );
+ if ($response->code != 200) {
+ printf("response code : %u\n", $response->code);
+ printf("response message : %s\n", $response->message);
+ }
+
+ $chunk_id++;
+ }
+}
diff --git a/ws.php b/ws.php
index 7867f8001..6c4c9a451 100644
--- a/ws.php
+++ b/ws.php
@@ -173,14 +173,24 @@ function ws_addDefaultMethods( $arr )
);
$service->addMethod(
+ 'pwg.images.addChunk',
+ 'ws_images_add_chunk',
+ array(
+ 'data' => array(),
+ 'original_sum' => array(),
+ 'type' => array(),
+ 'position' => array(),
+ ),
+ 'POST method only. For admin only.'
+ );
+
+
+ $service->addMethod(
'pwg.images.add',
'ws_images_add',
array(
- 'file_content' => array(),
'file_sum' => array(),
- 'thumbnail_content' => array(),
'thumbnail_sum' => array(),
- 'high_content' => array('default' => null),
'high_sum' => array('default' => null),
'original_sum' => array(),
'name' => array('default' => null),