ZBUG-4409: Missing Blobs on a restored account from backup with S3 storage

Refactored according to review comments

Refactored

Refactoring the testing classes

Refactored according to the review comments

ZBUG-4409 Added Ldap attribute for JUNIT

Author: Gopal-Moolchandani

store/src/java-test/com/zimbra/cs/account/MockProvisioning.java

@@ -85,6 +85,7 @@ public MockProvisioning() {
         attrs.put(A_zimbraSmtpPort, "7025");
         attrs.put(A_zimbraLowestSupportedAuthVersion, "1");
         attrs.put(A_zimbraSSLPrivateKey, "-----BEGIN PRIVATE KEY-----MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDj5CdJz5QNpk7skWqv7lC7gaymGiqbpEXAe8za1JTgjKkL0gzQNM79aGr+lnYKXU+rto15QT9uDyvVnY/iPlpeLtCEin9OIEccLsPivG0R8gL8O2HGF86ns+ZNbjjJVetr/qk1mFN/91qMfOhL/F6tBR5zSIVdCE4bdqCqpq4HywEaKyXxCwU4bKlmawrLRZeITiMQ6Je/VsFvz2wj7yrlCH5HKtoyaNuLR9KH1HHbFB/p9JCK9/qZpq5p4vNXr0fGs3PFQAfhau2ySmo1bEhYDIs3/nBLrXP3OHQqfEPEE1R7BrcEBBtUaY8t5JSduOQRr63qDyGjXnu+xcwA46v/AgMBAAECggEAUa57LoeKZ4IOk9hjRv/CTBLkkPyb/QFaRu2YtW6wlfOUu7nkAdSLxGRixTGkyX48ii16c9WhKI+jhINfCRaUSWG6N2d0zcnf8wgICgLDjUUTMNkP6HKsDYv7phE1pWR4Z1L1z1Hzy9Aa0nQKxwGD5bwJ+AQsWPYbGNjiKYhopD2/zbTnhhYCA9/aSf2vKUa9ITWpW7nbYK1dtOW+eX+CrBRBO54KQ0OlJBnzk2oZv7IOlp6PUck0HMurP/N/EV1lVwvsMaddy3osHrB1qjA/vSX+wYMaNRSyN8p9hRTAAPQKJKy3feXUrE/kzhV/MP0DYbHkV1SYDNt0nDarZ8wDAQKBgQD2r0UIdMmOwjGoR0y4/EXSvEf6+KP+t0GDEYfl9C5/Ei3BTFPU5bWg/1TEQJ5R4AsI6TchHE1k0o1tENZ09SO/9ESQPfRZHbbaMit64SAg3e8mvChEvVfaunDapOTuwiqzvTjc12plwXrzrkt2xtXLpWvZ8afWVaBhEZysieXvQQKBgQDsfzSc4AuUprqqLU7pyQGSy9zjv2Pt25aUdiyiVYawzDfcQly4x5b/b/NodQ2SG/5zN2/L+fvjHJvWcKsHx984siOGVMxEK5uUz00eKbVB/PZP+oT7MALStaI0U0gnwtnWgXCxMKl2QOfWnF3AMkF99xKywsuyFIg6ojoE9IYLPwKBgQDzxmi13pOAXC+uWCdddw+ZHS8UuLl3calv2NcvS4rXUCOfLcp6TTacDza5ahIKXxkIiU9NjSZ+SAQyj70ef1IA02ceE9twZYjZP1Lwb6DMWgWHhdFVfLdhE3WK3ADQYVjJnmie9NHUFMtoHAm/KucEBEj8a26sxJlk0368ktmDAQKBgBGMctv9J/7UzF8aU5O3bZ118SMZLZIVzDuh9Tfqfr8ZuD9o0TaI4OR9ayNiJCqmVyA3id0p5I36rnmgDKDcLO0pEsfB/RJF5hqJs2A8mg2WdrSCk2GMM3ltLucREvaYV8+59SHAyaJTuKBNJAvB7ugo8ENBfxnsuhsXtJRvjI7DAoGAG7HrReM3cX8k8jqopY8xlT03Q372v07PL8fs2aOP+zsA580IdDw0Xvea+dmykCzpj3DmBci+TOE7DYb1SW6+NPN4FOP6o6TFogrgNa/0LAWXPuct+1e5vy9F1/jFpvNbbD8uTJKdJWGKO78wLVybn9gTJ95ZZcBfytQMyXpqysk=-----END PRIVATE KEY-----");
+        attrs.put(A_zimbraMailboxMoveTempDir,System.getProperty("java.io.tmpdir")+ "/" + "testFileUtil");
         localhost = new Server("localhost", "localhost", attrs, Collections.<String, Object>emptyMap(), this);
         servers.put("localhost", localhost);
         try {

store/src/java-test/com/zimbra/cs/store/MockStoreManager.java

@@ -27,6 +27,7 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import com.zimbra.cs.volume.Volume;
 import org.apache.commons.io.output.ByteArrayOutputStream;
 
 import com.google.common.io.ByteStreams;
@@ -106,6 +107,11 @@ public StagedBlob stage(InputStream data, long actualSize, Mailbox mbox) throws
         return new MockStagedBlob(mbox, ByteStreams.toByteArray(data));
     }
 
+    @Override
+    public StagedBlob stage(InputStream data, long actualSize, Mailbox mbox, Volume volume) throws IOException {
+        return new MockStagedBlob(mbox, ByteStreams.toByteArray(data));
+    }
+
     @Override
     public StagedBlob stage(Blob blob, Mailbox mbox) {
         return new MockStagedBlob(mbox, ((MockBlob) blob).content);

store/src/java-test/com/zimbra/cs/store/StoreManagerAfterReset.java

@@ -18,6 +18,7 @@
 
 import com.zimbra.common.service.ServiceException;
 import com.zimbra.cs.mailbox.Mailbox;
+import com.zimbra.cs.volume.Volume;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -59,6 +60,11 @@ public StagedBlob stage(InputStream data, long actualSize, Mailbox mbox) throws
         return null;
     }
 
+    @Override
+    public StagedBlob stage(InputStream data, long actualSize, Mailbox mbox, Volume volume) throws IOException, ServiceException {
+        throw ServiceException.UNSUPPORTED();
+    }
+
     @Override
     public StagedBlob stage(Blob blob, Mailbox mbox) throws IOException, ServiceException {
         return null;

store/src/java/com/zimbra/cs/store/StoreManager.java

@@ -29,6 +29,7 @@
 import com.zimbra.cs.store.file.FileBlobStore;
 import com.zimbra.cs.store.helper.ClassHelper;
 import com.zimbra.cs.util.Zimbra;
+import com.zimbra.cs.volume.Volume;
 import com.zimbra.cs.volume.VolumeManager;
 
 import java.io.IOException;
@@ -46,6 +47,7 @@ public abstract class StoreManager {
      */
     private static short currentVolumeId;
     private static Integer diskStreamingThreshold;
+    private static final int actualSizeforUnstagedBlob = -1;
 
 
     public static StoreManager getInstance () {
@@ -292,6 +294,20 @@ public abstract Blob storeIncoming(InputStream data, boolean storeAsIs)
     public abstract StagedBlob stage(InputStream data, long actualSize, Mailbox mbox)
     throws IOException, ServiceException;
 
+    /**
+     * Stage an incoming <code>InputStream</code> to an
+     * appropriate place for subsequent storage in a <code>Mailbox</code> via
+     * {@link #link(StagedBlob, Mailbox, int, int)} or {@link #renameTo}.
+     *
+     * @param data the data stream
+     * @param actualSize the content size, or {@code -1} if the content size is not available
+     * @param callback callback, or {@code null}
+     * @param mbox the mailbox
+     * @param volume the volume
+     */
+    public abstract StagedBlob stage(InputStream data, long actualSize, Mailbox mbox, Volume volume)
+            throws IOException, ServiceException;
+
     /**
      * Stage an incoming <code>InputStream</code> to an
      * appropriate place for subsequent storage in a <code>Mailbox</code> via
@@ -303,7 +319,22 @@ public abstract StagedBlob stage(InputStream data, long actualSize, Mailbox mbox
      */
     public StagedBlob stage(InputStream data, Mailbox mbox)
     throws IOException, ServiceException {
-        return stage(data, -1, mbox);
+        return stage(data, actualSizeforUnstagedBlob, mbox);
+    }
+
+    /**
+     * Stage an incoming <code>InputStream</code> to an
+     * appropriate place for subsequent storage in a <code>Mailbox</code> via
+     * {@link #link(StagedBlob, Mailbox, int, int)} or {@link #renameTo}.
+     *
+     * @param data the data stream
+     * @param callback callback, or {@code null}
+     * @param mbox the mailbox
+     * @param volume the volume
+     */
+    public StagedBlob stage(InputStream data, Mailbox mbox, Volume volume)
+            throws IOException, ServiceException {
+        return stage(data, actualSizeforUnstagedBlob, mbox, volume);
     }
 
     /**

store/src/java/com/zimbra/cs/store/external/ExternalBlobIO.java

@@ -38,6 +38,21 @@ public interface ExternalBlobIO {
      */
     String writeStreamToStore(InputStream in, long actualSize, Mailbox mbox) throws IOException, ServiceException;
 
+    /**
+     * Write data to blob store
+     *
+     * @param in:         InputStream containing data to be written
+     * @param actualSize: size of data in stream, or -1 if size is unknown. To be used by implementor for optimization where possible
+     * @param mbox:       Mailbox which contains the blob. Can optionally be used by store for partitioning
+     * @param destVolId:  Destination Volume id on which we need to store the blob
+     * @return locator string for the stored blob, unique identifier created by storage protocol
+     * @throws IOException
+     * @throws ServiceException
+     */
+    default String writeStreamToStore(InputStream in, long actualSize, Mailbox mbox, short destVolId) throws IOException, ServiceException {
+        throw ServiceException.UNSUPPORTED();
+    }
+
     /**
      * Create an input stream for reading data from blob store
      * @param locator: identifier string for the blob as returned from write operation

store/src/java/com/zimbra/cs/store/external/ExternalStoreManager.java

@@ -44,6 +44,7 @@
 import com.zimbra.cs.store.StagedBlob;
 import com.zimbra.cs.store.StoreManager;
 import com.zimbra.cs.store.file.VolumeMailboxBlob;
+import com.zimbra.cs.volume.Volume;
 
 /**
  * Abstract base class for external store integration.
@@ -272,6 +273,31 @@ public StagedBlob stage(Blob blob, Mailbox mbox) throws IOException, ServiceExce
         }
     }
 
+    public StagedBlob stage(Blob blob, Mailbox mbox, Volume volume) throws IOException, ServiceException {
+        if (supports(StoreFeature.RESUMABLE_UPLOAD) && blob instanceof ExternalUploadedBlob) {
+            ZimbraLog.store.debug("blob already uploaded, just need to commit");
+            String locator = ((ExternalResumableUpload) this).finishUpload((ExternalUploadedBlob) blob);
+            if (null != locator) {
+                ZimbraLog.store.debug("wrote to locator %s", locator);
+                localCache.put(locator, getContent(blob));
+            } else {
+                ZimbraLog.store.warn("blob staging returned null locator");
+            }
+            return new ExternalStagedBlob(mbox, blob.getDigest(), blob.getRawSize(), locator);
+        } else {
+            InputStream is = getContent(blob);
+            try {
+                StagedBlob staged = stage(is, blob.getRawSize(), mbox, volume);
+                if (null != staged && null != staged.getLocator()) {
+                    localCache.put(staged.getLocator(), getContent(blob));
+                }
+                return staged;
+            } finally {
+                ByteUtil.closeStream(is);
+            }
+        }
+    }
+
     @Override
     public StagedBlob stage(InputStream in, long actualSize, Mailbox mbox) throws ServiceException, IOException {
         if (actualSize < 0) {
@@ -303,7 +329,35 @@ public StagedBlob stage(InputStream in, long actualSize, Mailbox mbox) throws Se
         }
     }
 
+    public StagedBlob stage(InputStream in, long actualSize, Mailbox mbox, Volume volume) throws ServiceException, IOException {
+        if (actualSize < 0) {
+            Blob blob = storeIncoming(in);
+            try {
+                return stage(blob, mbox, volume);
+            } finally {
+                quietDelete(blob);
+            }
+        }
+        MessageDigest digest;
+        try {
+            digest = MessageDigest.getInstance("SHA-256");
+        } catch (NoSuchAlgorithmException e) {
+            throw ServiceException.FAILURE("SHA-256 digest not found", e);
+        }
+        ByteUtil.PositionInputStream pin = new ByteUtil.PositionInputStream(new DigestInputStream(in, digest));
 
+        try {
+            String locator = writeStreamToStore(pin, actualSize, mbox, volume.getId());
+            if (null != locator) {
+                ZimbraLog.store.debug("wrote to locator %s",locator);
+            } else {
+                ZimbraLog.store.warn("blob staging returned null locator");
+            }
+            return new ExternalStagedBlob(mbox, ByteUtil.encodeFSSafeBase64(digest.digest()), pin.getPosition(), locator);
+        } catch (IOException e) {
+            throw ServiceException.FAILURE("unable to stage blob", e);
+        }
+    }
     @Override
     public Blob storeIncoming(InputStream data, boolean storeAsIs) throws IOException,
     ServiceException {

store/src/java/com/zimbra/cs/store/file/FileBlobStore.java

@@ -118,6 +118,12 @@ public VolumeStagedBlob stage(InputStream in, long actualSize, Mailbox mbox)
         return new VolumeStagedBlob(mbox, (VolumeBlob) blob).markStagedDirectly();
     }
 
+    @Override
+    public StagedBlob stage(InputStream data, long actualSize, Mailbox mbox, Volume volume) throws IOException, ServiceException {
+        // mailbox store is on the same volume as incoming directory, so no need to stage the blob
+        throw ServiceException.FAILURE("Operation can not be completed because the required StoreManager is not available", null);
+    }
+
     @Override
     public VolumeStagedBlob stage(Blob blob, Mailbox mbox) throws IOException {
         // mailbox store is on the same volume as incoming directory, so no need to stage the blob