bug 941 fixed: to be able to upload heavy photo, chunk the files, send parts

one by one, and then pwg.images.add merge chunks together. Now big uploads
works and you can even have a fine progress bar on client side.


git-svn-id: http://piwigo.org/svn/branches/2.0@3192 68402e56-0260-453c-a942-63ccdbb3a9ee

3 files changed, 206 insertions, 44 deletions

diff --git a/include/ws_functions.inc.php b/include/ws_functions.inc.php
index a85fa2e4b..7c1d52580 100644
--- a/include/ws_functions.inc.php
+++ b/include/ws_functions.inc.php
@@ -879,6 +879,98 @@ UPDATE '.IMAGES_TABLE.'
   return $affected_rows;
 }
 
+function ws_images_add_chunk($params, &$service)
+{
+  // data
+  // original_sum
+  // type {thumb, file, high}
+  // position
+  
+  if (!is_admin() || is_adviser() )
+  {
+    return new PwgError(401, 'Access denied');
+  }
+
+  $upload_dir = PHPWG_ROOT_PATH.'upload/buffer';
+
+  // create the upload directory tree if not exists
+  if (!is_dir($upload_dir)) {
+    umask(0000);
+    $recursive = true;
+    if (!@mkdir($upload_dir, 0777, $recursive))
+    {
+      return new PwgError(500, 'error during buffer directory creation');
+    }
+  }
+
+  if (!is_writable($upload_dir))
+  {
+    // last chance to make the directory writable
+    @chmod($upload_dir, 0777);
+
+    if (!is_writable($upload_dir))
+    {
+      return new PwgError(500, 'buffer directory has no write access');
+    }
+  }
+
+  secure_directory($upload_dir);
+
+  $filename = sprintf(
+    '%s-%s-%05u.block',
+    $params['original_sum'],
+    $params['type'],
+    $params['position']
+    );
+
+  $bytes_written = file_put_contents(
+    $upload_dir.'/'.$filename,
+    $params['data']
+    );
+
+  if (false === $bytes_written) {
+    return new PwgError(
+      500,
+      'an error has occured while writting chunk '.$params['position'].' for '.$params['type']
+      );
+  }
+}
+
+function merge_chunks($output_filepath, $original_sum, $type)
+{
+  ws_logfile('[merge_chunks] input parameter $output_filepath : '.$output_filepath);
+
+  $upload_dir = PHPWG_ROOT_PATH.'upload/buffer';
+  $pattern = '/'.$original_sum.'-'.$type.'/';
+  $chunks = array();
+  
+  if ($handle = opendir($upload_dir))
+  {
+    while (false !== ($file = readdir($handle)))
+    {
+      if (preg_match($pattern, $file))
+      {
+        ws_logfile($file);
+        array_push($chunks, $upload_dir.'/'.$file);
+      }
+    }
+    closedir($handle);
+  }
+
+  sort($chunks);
+  
+  $string = null;
+  foreach ($chunks as $chunk) {
+    $string.= file_get_contents($chunk);
+    unlink($chunk);
+  }
+  if (!file_put_contents($output_filepath, base64_decode($string)))
+  {
+    return new PwgError(500, 'error while merging chunks for '.$output_filepath);
+  }
+  
+}
+
 function ws_images_add($params, &$service)
 {
   global $conf;
@@ -956,13 +1048,8 @@ SELECT
   $filename_wo_ext = $date_string.'-'.$random_string;
   $file_path = $upload_dir.'/'.$filename_wo_ext.'.jpg';
 
-  // dump the photo file
-  $fh_file = fopen($file_path, 'w');
-  if (!fwrite($fh_file, base64_decode($params['file_content'])))
-  {
-    return new PwgError(500, 'error while writing file');
-  }
-  fclose($fh_file);
+  // merge the photo file
+  merge_chunks($file_path, $params['original_sum'], 'file');
   chmod($file_path, 0644);
 
   // check dumped file md5sum against expected md5sum
@@ -1005,13 +1092,8 @@ SELECT
     'jpg'
     );
 
-  // dump the thumbnail
-  $fh_thumbnail = fopen($thumbnail_path, 'w');
-  if (!fwrite($fh_thumbnail, base64_decode($params['thumbnail_content'])))
-  {
-    return new PwgError(500, 'error while writing thumbnail');
-  }
-  fclose($fh_thumbnail);
+  // merge the thumbnail
+  merge_chunks($thumbnail_path, $params['original_sum'], 'thumb');
   chmod($thumbnail_path, 0644);
 
   // check dumped thumbnail md5
@@ -1021,7 +1103,7 @@ SELECT
   }
 
   // high resolution
-  if (isset($params['high_content']))
+  if (isset($params['high_sum']))
   {
     // high resolution directory is a subdirectory of the photo file, hard
     // coded "pwg_high"
@@ -1055,13 +1137,8 @@ SELECT
       'jpg'
       );
 
-    // dump the high resolution file
-    $fh_high = fopen($high_path, 'w');
-    if (!fwrite($fh_high, base64_decode($params['high_content'])))
-    {
-      return new PwgError(500, 'error while writing high');
-    }
-    fclose($fh_high);
+    // merge the high resolution file
+    merge_chunks($high_path, $params['original_sum'], 'high');
     chmod($high_path, 0644);
 
     // check dumped thumbnail md5
@@ -1104,7 +1181,7 @@ SELECT
     }
   }
 
-  if (isset($params['high_content']))
+  if (isset($params['high_sum']))
   {
     $insert['has_high'] = 'true';
     $insert['high_filesize'] = $high_filesize;
@@ -1646,4 +1723,15 @@ SELECT
     update_category($cat_ids);
   }
 }
+
+function ws_logfile($string)
+{
+  return true;
+  
+  file_put_contents(
+    '/tmp/piwigo_ws.log',
+    '['.date('c').'] '.$string."\n",
+    FILE_APPEND
+    );
+}
 ?>
diff --git a/tools/piwigo_remote.pl b/tools/piwigo_remote.pl
index 6cd558cd1..91142f25a 100644
--- a/tools/piwigo_remote.pl
+++ b/tools/piwigo_remote.pl
@@ -1,5 +1,17 @@
 #!/usr/bin/perl
 
+####
+# Usage examples
+#
+# time perl piwigo_remote.pl \
+#   --action=pwg.images.add \
+#   --file=erwann_rocher-web.jpg \
+#   --thumb=erwann_rocher-thumb.jpg \
+#   --high=erwann_rocher-high.jpg \
+#   --original=erwann_rocher-high.jpg \
+#   --define categories=9 \
+#   --chunk_size=200_000
+
 use strict;
 use warnings;
 
@@ -7,22 +19,24 @@ use JSON;
 use LWP::UserAgent;
 use Getopt::Long;
 use Encode qw/is_utf8 decode/;
+use POSIX qw(ceil floor);
 
 my %opt = ();
 GetOptions(
     \%opt,
-    qw/action=s file=s thumbnail=s high=s original=s categories=s define=s%/
+    qw/action=s file=s thumbnail=s high=s original=s categories=s chunk_size=i define=s%/
 );
 
 our $ua = LWP::UserAgent->new;
 $ua->cookie_jar({});
 
 my %conf;
-$conf{base_url} = 'http://localhost/~pierrick/piwigo/2.0';
+$conf{base_url} = 'http://localhost/piwigo/2.0';
 $conf{response_format} = 'json';
-$conf{username} = 'pierrick';
-$conf{password} = 'z0rglub';
+$conf{username} = 'plg';
+$conf{password} = 'plg';
 $conf{limit} = 10;
+$conf{chunk_size} = defined $opt{chunk_size} ? $opt{chunk_size} : 500_000;
 
 my $result = undef;
 my $query = undef;
@@ -48,26 +62,32 @@ if ($opt{action} eq 'pwg.images.add') {
     use Digest::MD5::File qw/file_md5_hex/;
     use File::Slurp;
 
-    my $original_sum = file_md5_hex($opt{original});
+    $form = {};
+    $form->{method} = 'pwg.images.add';
 
-    my $file_content = encode_base64(read_file($opt{file}));
-    my $file_sum = file_md5_hex($opt{file});
+    my $original_sum = file_md5_hex($opt{original});
+    $form->{original_sum} = $original_sum;
 
-    my $thumbnail_content = encode_base64(read_file($opt{thumbnail}));
-    my $thumbnail_sum = file_md5_hex($opt{thumbnail});
+    send_chunks(
+        filepath => $opt{file},
+        type => 'file',
+        original_sum => $original_sum,
+    );
+    $form->{file_sum} = file_md5_hex($opt{file});
 
-    $form = {
-        method => 'pwg.images.add',
+    send_chunks(
+        filepath => $opt{thumbnail},
+        type => 'thumb',
         original_sum => $original_sum,
-        file_sum => $file_sum,
-        file_content => $file_content,
-        thumbnail_sum => $thumbnail_sum,
-        thumbnail_content => $thumbnail_content,
-        categories => $opt{categories},
-    };
+    );
+    $form->{thumbnail_sum} = file_md5_hex($opt{thumbnail});
 
     if (defined $opt{high}) {
-        $form->{high_content} = encode_base64(read_file($opt{high}));
+        send_chunks(
+            filepath => $opt{high},
+            type => 'high',
+            original_sum => $original_sum,
+        );
         $form->{high_sum} = file_md5_hex($opt{high});
     }
 
@@ -231,3 +251,47 @@ sub pwg_ws_get_query {
 
     return $query;
 }
+
+sub send_chunks {
+    my %params = @_;
+
+    my $content = encode_base64(read_file($params{filepath}));
+    my $content_length = length($content);
+    my $nb_chunks = ceil($content_length / $conf{chunk_size});
+
+    my $chunk_pos = 0;
+    my $chunk_id = 1;
+    while ($chunk_pos < $content_length) {
+        my $chunk = substr(
+            $content,
+            $chunk_pos,
+            $conf{chunk_size}
+        );
+        $chunk_pos += $conf{chunk_size};
+
+        my $response = $ua->post(
+            $conf{base_url}.'/ws.php?format=json',
+            {
+                method => 'pwg.images.addChunk',
+                data => $chunk,
+                original_sum => $params{original_sum},
+                position => $chunk_id,
+                type => $params{type},
+            }
+        );
+
+        printf(
+            'chunk %05u of %05u for %s "%s"'."\n",
+            $chunk_id,
+            $nb_chunks,
+            $params{type},
+            $params{filepath}
+        );
+        if ($response->code != 200) {
+            printf("response code    : %u\n", $response->code);
+            printf("response message : %s\n", $response->message);
+        }
+
+        $chunk_id++;
+    }
+}
diff --git a/ws.php b/ws.php
index 7867f8001..6c4c9a451 100644
--- a/ws.php
+++ b/ws.php
@@ -173,14 +173,24 @@ function ws_addDefaultMethods( $arr )
     );
 
   $service->addMethod(
+    'pwg.images.addChunk',
+    'ws_images_add_chunk',
+    array(
+      'data' => array(),
+      'original_sum' => array(),
+      'type' => array(),
+      'position' => array(),
+      ),
+    'POST method only. For admin only.'
+    );
+  
+
+  $service->addMethod(
     'pwg.images.add',
     'ws_images_add',
     array(
-      'file_content' => array(),
       'file_sum' => array(),
-      'thumbnail_content' => array(),
       'thumbnail_sum' => array(),
-      'high_content' => array('default' => null),
       'high_sum' => array('default' => null),
       'original_sum' => array(),
       'name' => array('default' => null),