Source code

001/*
002 * This library is part of OpenCms -
003 * the Open Source Content Management System
004 *
005 * Copyright (c) Alkacon Software GmbH & Co. KG (https://www.alkacon.com)
006 *
007 * This library is free software; you can redistribute it and/or
008 * modify it under the terms of the GNU Lesser General Public
009 * License as published by the Free Software Foundation; either
010 * version 2.1 of the License, or (at your option) any later version.
011 *
012 * This library is distributed in the hope that it will be useful,
013 * but WITHOUT ANY WARRANTY; without even the implied warranty of
014 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
015 * Lesser General Public License for more details.
016 *
017 * For further information about Alkacon Software GmbH & Co. KG, please see the
018 * company website: https://www.alkacon.com
019 *
020 * For further information about OpenCms, please see the
021 * project website: https://www.opencms.org
022 *
023 * You should have received a copy of the GNU Lesser General Public
024 * License along with this library; if not, write to the Free Software
025 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
026 */
027
028package org.opencms.loader;
029
030import org.opencms.db.storage.CmsStorageBlobNotFoundException;
031import org.opencms.db.storage.s3.CmsGenericS3Client;
032import org.opencms.db.storage.s3.CmsS3ClientConfiguration;
033import org.opencms.db.storage.s3.CmsS3DeleteResult;
034import org.opencms.db.storage.s3.CmsS3ObjectMetadata;
035import org.opencms.db.storage.s3.I_CmsS3Client;
036import org.opencms.loader.imagecache.CmsImageCacheEntry;
037import org.opencms.loader.imagecache.I_CmsImageCacheAccessMetadataProvider;
038import org.opencms.util.CmsStringUtil;
039
040import java.io.OutputStream;
041import java.nio.charset.StandardCharsets;
042import java.time.Instant;
043import java.util.ArrayList;
044import java.util.Arrays;
045import java.util.HashMap;
046import java.util.HashSet;
047import java.util.LinkedHashMap;
048import java.util.List;
049import java.util.Map;
050import java.util.Set;
051import java.util.UUID;
052import java.util.concurrent.TimeUnit;
053
054import com.google.common.cache.Cache;
055import com.google.common.cache.CacheBuilder;
056
057/**
058 * S3 based storage for generated image cache entries.<p>
059 */
060public class CmsS3ImageCache implements I_CmsImageCache, I_CmsImageCacheAccessMetadataProvider {
061
062    /** Default and maximum S3 delete batch size. */
063    private static final int DEFAULT_DELETE_BATCH_SIZE = CmsGenericS3Client.MAX_DELETE_OBJECTS;
064
065    /** Maximum number of S3 object lengths kept locally. */
066    private static final int LENGTH_CACHE_MAX_SIZE = 10000;
067
068    /** Number of minutes after which a cached S3 object length expires. */
069    private static final int LENGTH_CACHE_EXPIRY_MINUTES = 10;
070
071    /** Locally cached S3 object lengths. */
072    private Cache<String, Long> m_lengthCache = CacheBuilder.newBuilder().maximumSize(
073        LENGTH_CACHE_MAX_SIZE).expireAfterAccess(LENGTH_CACHE_EXPIRY_MINUTES, TimeUnit.MINUTES).build();
074
075    /** Metadata obtained without an extra request while delivering S3 objects. */
076    private Cache<String, CmsImageCacheEntry> m_recentAccessMetadata = CacheBuilder.newBuilder().maximumSize(
077        LENGTH_CACHE_MAX_SIZE).expireAfterAccess(LENGTH_CACHE_EXPIRY_MINUTES, TimeUnit.MINUTES).build();
078
079    /** The S3 client. */
080    private I_CmsS3Client m_s3Client;
081
082    /**
083     * Creates a new image cache.<p>
084     *
085     * @param configuration the S3 configuration
086     * @throws Exception if the configured bucket can not be accessed
087     */
088    public CmsS3ImageCache(CmsS3ClientConfiguration configuration)
089    throws Exception {
090
091        initClient(new CmsGenericS3Client(configuration));
092    }
093
094    /**
095     * Creates a new image cache with a custom client.<p>
096     *
097     * @param s3Client the S3 client
098     * @throws Exception if the configured bucket can not be accessed
099     */
100    CmsS3ImageCache(I_CmsS3Client s3Client)
101    throws Exception {
102
103        initClient(s3Client);
104    }
105
106    /**
107     * @see org.opencms.loader.I_CmsImageCache#clear()
108     */
109    @Override
110    public void clear() throws Exception {
111
112        clear(DEFAULT_DELETE_BATCH_SIZE);
113    }
114
115    /**
116     * Clears the image cache using S3 multi-object delete requests.<p>
117     *
118     * @param deleteBatchSize the number of keys per request
119     * @throws Exception if listing or deleting fails
120     */
121    public void clear(int deleteBatchSize) throws Exception {
122
123        ensureInitialized();
124        if ((deleteBatchSize < 1) || (deleteBatchSize > CmsGenericS3Client.MAX_DELETE_OBJECTS)) {
125            throw new IllegalArgumentException(
126                "S3 delete batch size must be between 1 and " + CmsGenericS3Client.MAX_DELETE_OBJECTS + ".");
127        }
128        List<String> batch = new ArrayList<String>(deleteBatchSize);
129        List<Exception> failures = new ArrayList<Exception>();
130        try {
131            m_s3Client.visitObjects("", metadata -> {
132                if (isImageCacheObjectKey(metadata.getKey())) {
133                    batch.add(metadata.getKey());
134                    if (batch.size() == deleteBatchSize) {
135                        failures.addAll(deleteObjectKeys(batch).getFailures().values());
136                        batch.clear();
137                    }
138                }
139            });
140            if (!batch.isEmpty()) {
141                failures.addAll(deleteObjectKeys(batch).getFailures().values());
142            }
143            if (!failures.isEmpty()) {
144                Exception failure = new Exception("Unable to delete " + failures.size() + " S3 image cache entries.");
145                for (Exception suppressed : failures) {
146                    failure.addSuppressed(suppressed);
147                }
148                throw failure;
149            }
150        } finally {
151            m_lengthCache.invalidateAll();
152            m_recentAccessMetadata.invalidateAll();
153        }
154    }
155
156    /**
157     * @see org.opencms.loader.I_CmsImageCache#close()
158     */
159    @Override
160    public void close() throws Exception {
161
162        I_CmsS3Client client = m_s3Client;
163        m_s3Client = null;
164        m_lengthCache.invalidateAll();
165        m_recentAccessMetadata.invalidateAll();
166        if (client != null) {
167            client.close();
168        }
169    }
170
171    /**
172     * Deletes an image cache entry.<p>
173     *
174     * @param key the image cache key
175     * @throws Exception if deleting fails
176     */
177    public void delete(String key) throws Exception {
178
179        ensureInitialized();
180        String objectKey = getObjectKey(key);
181        try {
182            m_s3Client.deleteObject(objectKey);
183        } finally {
184            m_lengthCache.invalidate(objectKey);
185            m_recentAccessMetadata.invalidate(normalizeKey(key));
186        }
187    }
188
189    /**
190     * Deletes image cache entries with one S3 multi-object request.<p>
191     *
192     * @param keys the image cache keys
193     * @return the per-entry result using normalized image cache keys
194     * @throws Exception if the complete request fails
195     */
196    public CmsS3DeleteResult deleteBatch(List<String> keys) throws Exception {
197
198        ensureInitialized();
199        List<String> objectKeys = new ArrayList<String>(keys.size());
200        Map<String, String> objectToCacheKey = new HashMap<String, String>();
201        List<String> normalizedKeys = new ArrayList<String>(keys.size());
202        Set<String> uniqueKeys = new HashSet<String>();
203        for (String key : keys) {
204            String normalizedKey = normalizeKey(key);
205            if (!uniqueKeys.add(normalizedKey)) {
206                throw new IllegalArgumentException("Duplicate normalized image cache key: " + normalizedKey);
207            }
208            String objectKey = normalizedKey;
209            normalizedKeys.add(normalizedKey);
210            objectKeys.add(objectKey);
211            objectToCacheKey.put(objectKey, normalizedKey);
212        }
213        try {
214            CmsS3DeleteResult objectResult = m_s3Client.deleteObjects(objectKeys);
215            Map<String, Exception> failures = new LinkedHashMap<String, Exception>();
216            for (Map.Entry<String, Exception> failure : objectResult.getFailures().entrySet()) {
217                String cacheKey = objectToCacheKey.get(failure.getKey());
218                failures.put(cacheKey == null ? failure.getKey() : cacheKey, failure.getValue());
219            }
220            return new CmsS3DeleteResult(normalizedKeys, failures);
221        } finally {
222            for (String objectKey : objectKeys) {
223                m_lengthCache.invalidate(objectKey);
224            }
225            for (String normalizedKey : normalizedKeys) {
226                m_recentAccessMetadata.invalidate(normalizedKey);
227            }
228        }
229    }
230
231    /**
232     * @see org.opencms.loader.I_CmsImageCache#exists(java.lang.String)
233     */
234    public boolean exists(String key) throws Exception {
235
236        ensureInitialized();
237        String objectKey = getObjectKey(key);
238        Long cachedLength = m_lengthCache.getIfPresent(objectKey);
239        if (cachedLength != null) {
240            return true;
241        }
242        long length = m_s3Client.getObjectLengthIfExists(objectKey);
243        if (length >= 0) {
244            m_lengthCache.put(objectKey, Long.valueOf(length));
245            return true;
246        }
247        return false;
248    }
249
250    /**
251     * @see org.opencms.loader.I_CmsImageCache#existsAuthoritatively(java.lang.String)
252     */
253    @Override
254    public boolean existsAuthoritatively(String key) throws Exception {
255
256        ensureInitialized();
257        String objectKey = getObjectKey(key);
258        long length = m_s3Client.getObjectLengthIfExists(objectKey);
259        if (length >= 0) {
260            m_lengthCache.put(objectKey, Long.valueOf(length));
261            return true;
262        }
263        m_lengthCache.invalidate(objectKey);
264        m_recentAccessMetadata.invalidate(normalizeKey(key));
265        return false;
266    }
267
268    /**
269     * @see org.opencms.loader.I_CmsImageCache#getLength(java.lang.String)
270     */
271    public long getLength(String key) throws Exception {
272
273        ensureInitialized();
274        String objectKey = getObjectKey(key);
275        Long cachedLength = m_lengthCache.getIfPresent(objectKey);
276        if (cachedLength != null) {
277            return cachedLength.longValue();
278        }
279        try {
280            long length = m_s3Client.getObjectLength(objectKey);
281            m_lengthCache.put(objectKey, Long.valueOf(length));
282            return length;
283        } catch (CmsStorageBlobNotFoundException e) {
284            m_lengthCache.invalidate(objectKey);
285            m_recentAccessMetadata.invalidate(normalizeKey(key));
286            throw new CmsImageCacheEntryNotFoundException(key, e);
287        }
288    }
289
290    /**
291     * Loads metadata for a single image cache entry.<p>
292     *
293     * @param key the image cache key
294     * @return the object metadata using the normalized image cache key
295     * @throws Exception if the metadata can not be loaded
296     */
297    public CmsS3ObjectMetadata getMetadata(String key) throws Exception {
298
299        ensureInitialized();
300        String objectKey = getObjectKey(key);
301        CmsS3ObjectMetadata metadata = m_s3Client.getObjectMetadata(objectKey);
302        m_lengthCache.put(objectKey, Long.valueOf(metadata.getLength()));
303        CmsS3ObjectMetadata result = new CmsS3ObjectMetadata(
304            normalizeKey(key),
305            metadata.getLength(),
306            metadata.getLastModified(),
307            metadata.getRevision());
308        rememberAccessMetadata(result);
309        return result;
310    }
311
312    /**
313     * @see org.opencms.loader.imagecache.I_CmsImageCacheAccessMetadataProvider#getRecentAccessMetadata(java.lang.String)
314     */
315    @Override
316    public CmsImageCacheEntry getRecentAccessMetadata(String key) {
317
318        return m_recentAccessMetadata.getIfPresent(normalizeKey(key));
319    }
320
321    /**
322     * Renews an image cache entry by conditionally copying the S3 object onto itself.<p>
323     *
324     * @param key the image cache key
325     * @param expectedRevision the expected object revision
326     * @param renewalTime the requested renewal time
327     * @return {@code true} if the entry was renewed, or {@code false} if it no longer matched
328     * @throws Exception if renewal fails
329     */
330    public boolean renew(String key, String expectedRevision, Instant renewalTime) throws Exception {
331
332        ensureInitialized();
333        if (CmsStringUtil.isEmptyOrWhitespaceOnly(expectedRevision)) {
334            throw new IllegalArgumentException("S3 image cache renewal requires an expected object revision.");
335        }
336        if (renewalTime == null) {
337            throw new IllegalArgumentException("S3 image cache renewal requires a renewal time.");
338        }
339        String objectKey = getObjectKey(key);
340        boolean renewed = m_s3Client.renewObject(objectKey, expectedRevision, renewalTime);
341        m_recentAccessMetadata.invalidate(normalizeKey(key));
342        if (!renewed) {
343            m_lengthCache.invalidate(objectKey);
344        }
345        return renewed;
346    }
347
348    /**
349     * @see org.opencms.loader.I_CmsImageCache#supportsRangeDelivery()
350     */
351    public boolean supportsRangeDelivery() {
352
353        return true;
354    }
355
356    /**
357     * @see org.opencms.loader.I_CmsImageCache#visitEntries(org.opencms.loader.I_CmsImageCache.I_CmsImageCacheEntryVisitor)
358     */
359    @Override
360    public void visitEntries(I_CmsImageCacheEntryVisitor visitor) throws Exception {
361
362        visitEntries("", visitor);
363    }
364
365    /**
366     * @see org.opencms.loader.I_CmsImageCache#visitEntries(java.lang.String, org.opencms.loader.I_CmsImageCache.I_CmsImageCacheEntryVisitor)
367     */
368    @Override
369    public void visitEntries(String prefix, I_CmsImageCacheEntryVisitor visitor) throws Exception {
370
371        visitEntriesWithMetadata(prefix, metadata -> visitor.visit(metadata.getKey(), metadata.getLength()));
372    }
373
374    /**
375     * Visits image cache entries with metadata returned directly by the S3 listing.<p>
376     *
377     * @param visitor the metadata visitor
378     * @throws Exception if listing fails
379     */
380    public void visitEntriesWithMetadata(I_CmsS3Client.I_CmsS3ObjectMetadataVisitor visitor) throws Exception {
381
382        visitEntriesWithMetadata("", visitor);
383    }
384
385    /**
386     * Visits image cache entries with the given cache key prefix and metadata returned directly by the S3 listing.<p>
387     *
388     * @param prefix the image cache key prefix, or an empty string for all entries
389     * @param visitor the metadata visitor
390     * @throws Exception if listing fails
391     */
392    public void visitEntriesWithMetadata(String prefix, I_CmsS3Client.I_CmsS3ObjectMetadataVisitor visitor)
393    throws Exception {
394
395        ensureInitialized();
396        String objectPrefix = normalizeKey(prefix == null ? "" : prefix);
397        m_s3Client.visitObjects(objectPrefix, metadata -> {
398            String objectKey = metadata.getKey();
399            if (isImageCacheObjectKey(objectKey)) {
400                String key = objectKey;
401                m_lengthCache.put(objectKey, Long.valueOf(metadata.getLength()));
402                visitor.visit(
403                    new CmsS3ObjectMetadata(
404                        key,
405                        metadata.getLength(),
406                        metadata.getLastModified(),
407                        metadata.getRevision()));
408            }
409        });
410    }
411
412    /**
413     * @see org.opencms.loader.I_CmsImageCache#write(java.lang.String, byte[])
414     */
415    public void write(String key, byte[] content) throws Exception {
416
417        ensureInitialized();
418        String objectKey = getObjectKey(key);
419        m_s3Client.putObject(objectKey, content);
420        m_lengthCache.put(objectKey, Long.valueOf(content.length));
421        m_recentAccessMetadata.invalidate(normalizeKey(key));
422    }
423
424    /**
425     * @see org.opencms.loader.I_CmsImageCache#writeRangeTo(java.lang.String, long, long, java.io.OutputStream)
426     */
427    public void writeRangeTo(String key, long start, long length, OutputStream out) throws Exception {
428
429        ensureInitialized();
430        String objectKey = getObjectKey(key);
431        try {
432            CmsS3ObjectMetadata metadata = m_s3Client.writeObjectRangeToWithMetadata(objectKey, start, length, out);
433            Long completeLength = m_lengthCache.getIfPresent(objectKey);
434            rememberAccessMetadata(
435                new CmsS3ObjectMetadata(
436                    normalizeKey(key),
437                    completeLength == null ? metadata.getLength() : completeLength.longValue(),
438                    metadata.getLastModified(),
439                    metadata.getRevision()));
440        } catch (CmsStorageBlobNotFoundException e) {
441            m_lengthCache.invalidate(objectKey);
442            m_recentAccessMetadata.invalidate(normalizeKey(key));
443            throw new CmsImageCacheEntryNotFoundException(key, e);
444        } catch (Exception e) {
445            m_lengthCache.invalidate(objectKey);
446            m_recentAccessMetadata.invalidate(normalizeKey(key));
447            throw e;
448        }
449    }
450
451    /**
452     * @see org.opencms.loader.I_CmsImageCache#writeTo(java.lang.String, java.io.OutputStream)
453     */
454    public void writeTo(String key, OutputStream out) throws Exception {
455
456        ensureInitialized();
457        String objectKey = getObjectKey(key);
458        try {
459            CmsS3ObjectMetadata metadata = m_s3Client.writeObjectToWithMetadata(objectKey, out);
460            rememberAccessMetadata(
461                new CmsS3ObjectMetadata(
462                    normalizeKey(key),
463                    metadata.getLength(),
464                    metadata.getLastModified(),
465                    metadata.getRevision()));
466        } catch (CmsStorageBlobNotFoundException e) {
467            m_lengthCache.invalidate(objectKey);
468            m_recentAccessMetadata.invalidate(normalizeKey(key));
469            throw new CmsImageCacheEntryNotFoundException(key, e);
470        } catch (Exception e) {
471            m_lengthCache.invalidate(objectKey);
472            m_recentAccessMetadata.invalidate(normalizeKey(key));
473            throw e;
474        }
475    }
476
477    /**
478     * Initializes the S3 client.<p>
479     *
480     * @param s3Client the S3 client
481     * @throws Exception if the configured bucket can not be accessed
482     */
483    protected void initClient(I_CmsS3Client s3Client) throws Exception {
484
485        String healthCheckKey = ".opencms-healthcheck/" + UUID.randomUUID().toString();
486        byte[] healthCheckContent = "OpenCms image cache health check".getBytes(StandardCharsets.UTF_8);
487        boolean healthCheckStored = false;
488        Exception failure = null;
489        try {
490            s3Client.validateBucketAccess();
491            s3Client.putObject(healthCheckKey, healthCheckContent);
492            healthCheckStored = true;
493            if (!Arrays.equals(healthCheckContent, s3Client.getObject(healthCheckKey))) {
494                throw new IllegalStateException("S3 image cache health check returned different content.");
495            }
496        } catch (Exception e) {
497            failure = e;
498            throw e;
499        } finally {
500            if (healthCheckStored) {
501                try {
502                    s3Client.deleteObject(healthCheckKey);
503                } catch (Exception e) {
504                    if (failure != null) {
505                        failure.addSuppressed(e);
506                    } else {
507                        failure = e;
508                    }
509                }
510            }
511            if (failure != null) {
512                try {
513                    s3Client.close();
514                } catch (Exception closeException) {
515                    failure.addSuppressed(closeException);
516                }
517            }
518        }
519        if (failure != null) {
520            throw failure;
521        }
522        if ((m_s3Client != null) && (m_s3Client != s3Client)) {
523            m_s3Client.close();
524        }
525        m_s3Client = s3Client;
526        m_lengthCache.invalidateAll();
527    }
528
529    /** Deletes a batch of complete S3 object keys. */
530    private CmsS3DeleteResult deleteObjectKeys(List<String> objectKeys) throws Exception {
531
532        List<String> keys = new ArrayList<String>(objectKeys);
533        try {
534            return m_s3Client.deleteObjects(keys);
535        } finally {
536            for (String key : keys) {
537                m_lengthCache.invalidate(key);
538            }
539        }
540    }
541
542    /**
543     * Ensures the cache has been initialized.<p>
544     */
545    private void ensureInitialized() {
546
547        if (m_s3Client == null) {
548            throw new IllegalStateException("Image cache has not been initialized.");
549        }
550    }
551
552    /**
553     * Returns the S3 object key.<p>
554     *
555     * @param key the image cache key
556     * @return the object key
557     */
558    private String getObjectKey(String key) {
559
560        return normalizeKey(key);
561    }
562
563    /**
564     * Checks whether an object key belongs to this image cache.<p>
565     *
566     * @param objectKey the object key
567     * @return <code>true</code> if the object key belongs to this image cache
568     */
569    private boolean isImageCacheObjectKey(String objectKey) {
570
571        return !objectKey.isEmpty();
572    }
573
574    /** Normalizes an image cache key. */
575    private String normalizeKey(String key) {
576
577        String normalizedKey = key;
578        while (normalizedKey.startsWith("/")) {
579            normalizedKey = normalizedKey.substring(1);
580        }
581        return normalizedKey;
582    }
583
584    /** Stores S3 response metadata for access-triggered renewal. */
585    private void rememberAccessMetadata(CmsS3ObjectMetadata metadata) {
586
587        if ((metadata.getLastModified() != null) && CmsStringUtil.isNotEmptyOrWhitespaceOnly(metadata.getRevision())) {
588            m_recentAccessMetadata.put(
589                normalizeKey(metadata.getKey()),
590                new CmsImageCacheEntry(
591                    normalizeKey(metadata.getKey()),
592                    metadata.getLength(),
593                    metadata.getLastModified(),
594                    metadata.getRevision()));
595        }
596    }
597}