001/*
002 * This library is part of OpenCms -
003 * the Open Source Content Management System
004 *
005 * Copyright (c) Alkacon Software GmbH & Co. KG (https://www.alkacon.com)
006 *
007 * This library is free software; you can redistribute it and/or
008 * modify it under the terms of the GNU Lesser General Public
009 * License as published by the Free Software Foundation; either
010 * version 2.1 of the License, or (at your option) any later version.
011 *
012 * This library is distributed in the hope that it will be useful,
013 * but WITHOUT ANY WARRANTY; without even the implied warranty of
014 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
015 * Lesser General Public License for more details.
016 *
017 * For further information about Alkacon Software, please see the
018 * company website: https://www.alkacon.com
019 *
020 * For further information about OpenCms, please see the
021 * project website: https://www.opencms.org
022 *
023 * You should have received a copy of the GNU Lesser General Public
024 * License along with this library; if not, write to the Free Software
025 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
026 */
027
028package org.opencms.db.storage;
029
030import org.opencms.db.CmsDbContext;
031import org.opencms.db.storage.s3.CmsGenericS3Client;
032import org.opencms.db.storage.s3.CmsS3ClientConfiguration;
033import org.opencms.db.storage.s3.I_CmsS3Client;
034import org.opencms.file.I_CmsFileContentStreamHandler;
035
036import java.io.OutputStream;
037import java.nio.charset.StandardCharsets;
038import java.util.Arrays;
039import java.util.UUID;
040
041/**
042 * S3-compatible object storage backend for the OpenCms deduplicating media storage engine (DMSE).<p>
043 *
044 * This implementation stores binary content in an S3-compatible object storage.
045 * To ensure high performance for read and write operations, it is highly recommended
046 * to use a <b>local object storage</b> (e.g., RustFS or an on-premise S3 appliance)
047 * within the same network as the OpenCms application server. Using remote cloud
048 * storage may introduce latency that significantly impacts VFS performance.<p>
049 *
050 * Content is addressed by its unique hash to support deduplication across
051 * different resources.<p>
052 *
053 * @see org.opencms.db.storage.CmsStorageManager
054 */
055public class CmsS3Storage extends A_CmsStorage implements I_CmsEnumerableStorage, I_CmsStorageDelivery {
056
057    /** The type name of the storage implementation. */
058    public static final String STORAGE_TYPE = "s3";
059
060    /** The S3 client. */
061    private final I_CmsS3Client m_s3Client;
062
063    /** The client configuration. */
064    private final CmsS3ClientConfiguration m_configuration;
065
066    /** The storage identifier. */
067    private final String m_identifier;
068
069    /** Whether path-style access should be used. */
070    private final boolean m_pathStyle;
071
072    /**
073     * Creates a new S3 storage.
074     *
075     * @param name the configured backend id used as storage identifier
076     * @param configuration the S3 client configuration
077     */
078    public CmsS3Storage(String name, CmsS3ClientConfiguration configuration) {
079
080        m_configuration = configuration;
081        m_identifier = name;
082        m_pathStyle = configuration.isPathStyle();
083        m_s3Client = new CmsGenericS3Client(configuration);
084    }
085
086    /**
087     * Creates a new S3 storage.
088     *
089     * @param name the configured backend id used as storage identifier
090     * @param endpoint the S3 endpoint
091     * @param bucketName the bucket name
092     * @param accessKey the access key
093     * @param secretKey the secret key
094     * @param pathStyle whether path-style access should be used
095     */
096    public CmsS3Storage(
097        String name,
098        String endpoint,
099        String bucketName,
100        String accessKey,
101        String secretKey,
102        boolean pathStyle) {
103
104        this(name, CmsS3ClientConfiguration.createDefault(endpoint, bucketName, accessKey, secretKey, pathStyle));
105    }
106
107    /**
108     * Creates a new S3 storage.
109     *
110     * @param name the configured backend id used as storage identifier
111     * @param endpoint the S3 endpoint
112     * @param bucketName the bucket name
113     * @param accessKey the access key
114     * @param secretKey the secret key
115     * @param pathStyle whether path-style access should be used
116     * @param connectionTimeout the connection timeout in milliseconds
117     * @param socketTimeout the socket timeout in milliseconds
118     * @param apiCallAttemptTimeout the timeout for a single API call attempt in milliseconds
119     * @param apiCallTimeout the timeout for the complete API call in milliseconds
120     * @param maxRetries the maximum number of retries
121     */
122    public CmsS3Storage(
123        String name,
124        String endpoint,
125        String bucketName,
126        String accessKey,
127        String secretKey,
128        boolean pathStyle,
129        int connectionTimeout,
130        int socketTimeout,
131        int apiCallAttemptTimeout,
132        int apiCallTimeout,
133        int maxRetries) {
134
135        this(
136            name,
137            endpoint,
138            bucketName,
139            accessKey,
140            secretKey,
141            pathStyle,
142            CmsS3ClientConfiguration.DEFAULT_REGION,
143            connectionTimeout,
144            socketTimeout,
145            apiCallAttemptTimeout,
146            apiCallTimeout,
147            maxRetries);
148    }
149
150    /**
151     * Creates a new S3 storage.
152     *
153     * @param name the configured backend id used as storage identifier
154     * @param endpoint the S3 endpoint
155     * @param bucketName the bucket name
156     * @param accessKey the access key
157     * @param secretKey the secret key
158     * @param pathStyle whether path-style access should be used
159     * @param region the AWS region
160     * @param connectionTimeout the connection timeout in milliseconds
161     * @param socketTimeout the socket timeout in milliseconds
162     * @param apiCallAttemptTimeout the timeout for a single API call attempt in milliseconds
163     * @param apiCallTimeout the timeout for the complete API call in milliseconds
164     * @param maxRetries the maximum number of retries
165     */
166    public CmsS3Storage(
167        String name,
168        String endpoint,
169        String bucketName,
170        String accessKey,
171        String secretKey,
172        boolean pathStyle,
173        String region,
174        int connectionTimeout,
175        int socketTimeout,
176        int apiCallAttemptTimeout,
177        int apiCallTimeout,
178        int maxRetries) {
179
180        this(
181            name,
182            new CmsS3ClientConfiguration(
183                endpoint,
184                bucketName,
185                accessKey,
186                secretKey,
187                pathStyle,
188                region,
189                connectionTimeout,
190                socketTimeout,
191                apiCallAttemptTimeout,
192                apiCallTimeout,
193                maxRetries));
194    }
195
196    /**
197     * Creates a new S3 storage with a custom client.<p>
198     *
199     * @param name the configured backend id used as storage identifier
200     * @param bucketName the bucket name
201     * @param pathStyle whether path-style access should be used
202     * @param s3Client the S3 client
203     */
204    CmsS3Storage(String name, String bucketName, boolean pathStyle, I_CmsS3Client s3Client) {
205
206        m_configuration = CmsS3ClientConfiguration.createDefault(
207            "http://localhost",
208            bucketName,
209            "access",
210            "secret",
211            pathStyle);
212        m_identifier = name;
213        m_pathStyle = pathStyle;
214        m_s3Client = s3Client;
215    }
216
217    /**
218     * @see org.opencms.db.storage.I_CmsStorage#close()
219     */
220    @Override
221    public void close() throws Exception {
222
223        m_s3Client.close();
224    }
225
226    /**
227     * @see org.opencms.db.storage.I_CmsStorage#deleteContent(org.opencms.db.CmsDbContext, java.lang.String)
228     */
229    @Override
230    public void deleteContent(CmsDbContext dbc, String hash) throws Exception {
231
232        String s3Key = getHashedPath(hash);
233        m_s3Client.deleteObject(s3Key);
234    }
235
236    /**
237     * @see org.opencms.db.storage.I_CmsStorage#getStorageIdentifier()
238     */
239    @Override
240    public String getStorageIdentifier() {
241
242        return m_identifier;
243    }
244
245    /**
246     * Returns whether path-style access is configured.
247     *
248     * @return true if path-style access is configured
249     */
250    public boolean isPathStyle() {
251
252        return m_pathStyle;
253    }
254
255    /**
256     * @see org.opencms.db.storage.I_CmsStorage#loadContent(org.opencms.db.CmsDbContext, java.lang.String)
257     */
258    @Override
259    public byte[] loadContent(CmsDbContext dbc, String hash) throws Exception {
260
261        String s3Key = getHashedPath(hash);
262        try {
263            byte[] result = m_s3Client.getObject(s3Key);
264            if (result == null) {
265                throw new CmsStorageBlobNotFoundException(
266                    Messages.get().getBundle().key(Messages.ERR_STORAGE_BLOB_MISSING_2, getStorageIdentifier(), hash));
267            }
268            return result;
269        } catch (CmsStorageBlobNotFoundException e) {
270            throw new CmsStorageBlobNotFoundException(
271                Messages.get().getBundle().key(Messages.ERR_STORAGE_BLOB_MISSING_2, getStorageIdentifier(), hash),
272                e);
273        }
274    }
275
276    /**
277     * @see I_CmsStorage#loadContentFrom(CmsDbContext, String, I_CmsFileContentStreamHandler)
278     */
279    @Override
280    public void loadContentFrom(CmsDbContext dbc, String hash, I_CmsFileContentStreamHandler handler) throws Exception {
281
282        String s3Key = getHashedPath(hash);
283        try {
284            m_s3Client.readObjectFrom(s3Key, handler);
285        } catch (CmsStorageBlobNotFoundException e) {
286            throw new CmsStorageBlobNotFoundException(
287                Messages.get().getBundle().key(Messages.ERR_STORAGE_BLOB_MISSING_2, getStorageIdentifier(), hash),
288                e);
289        }
290    }
291
292    /**
293     * @see I_CmsStorage#loadContentTo(CmsDbContext, String, OutputStream)
294     */
295    @Override
296    public void loadContentTo(CmsDbContext dbc, String hash, OutputStream out) throws Exception {
297
298        String s3Key = getHashedPath(hash);
299        try {
300            m_s3Client.writeObjectTo(s3Key, out);
301        } catch (CmsStorageBlobNotFoundException e) {
302            throw new CmsStorageBlobNotFoundException(
303                Messages.get().getBundle().key(Messages.ERR_STORAGE_BLOB_MISSING_2, getStorageIdentifier(), hash),
304                e);
305        }
306    }
307
308    /**
309     * @see org.opencms.db.storage.I_CmsStorage#storeContent(org.opencms.db.CmsDbContext, java.lang.String, byte[])
310     */
311    @Override
312    public void storeContent(CmsDbContext dbc, String hash, byte[] content) throws Exception {
313
314        if (content == null) {
315            throw new IllegalArgumentException(Messages.get().getBundle().key(Messages.ERR_STORAGE_CONTENT_NULL_0));
316        }
317        String s3Key = getHashedPath(hash);
318        if (!m_s3Client.exists(s3Key)) {
319            m_s3Client.putObject(s3Key, content);
320        }
321    }
322
323    /**
324     * @see org.opencms.db.storage.I_CmsStorageDelivery#streamRangeTo(org.opencms.db.CmsDbContext, java.lang.String, long, long, java.io.OutputStream)
325     */
326    @Override
327    public void streamRangeTo(CmsDbContext dbc, String hash, long start, long length, OutputStream out)
328    throws Exception {
329
330        String s3Key = getHashedPath(hash);
331        try {
332            m_s3Client.writeObjectRangeTo(s3Key, start, length, out);
333        } catch (CmsStorageBlobNotFoundException e) {
334            throw new CmsStorageBlobNotFoundException(
335                Messages.get().getBundle().key(Messages.ERR_STORAGE_BLOB_MISSING_2, getStorageIdentifier(), hash),
336                e);
337        }
338    }
339
340    /**
341     * @see org.opencms.db.storage.I_CmsStorageDelivery#streamTo(org.opencms.db.CmsDbContext, java.lang.String, java.io.OutputStream)
342     */
343    @Override
344    public void streamTo(CmsDbContext dbc, String hash, OutputStream out) throws Exception {
345
346        loadContentTo(dbc, hash, out);
347    }
348
349    /**
350     * @see org.opencms.db.storage.I_CmsStorageDelivery#supportsRangeDelivery()
351     */
352    @Override
353    public boolean supportsRangeDelivery() {
354
355        return true;
356    }
357
358    /**
359     * @see org.opencms.db.storage.I_CmsStorage#validateAvailable(org.opencms.db.CmsDbContext)
360     */
361    @Override
362    public void validateAvailable(CmsDbContext dbc) throws Exception {
363
364        String key = ".opencms-healthcheck/" + UUID.randomUUID().toString();
365        byte[] content = "OpenCms storage health check".getBytes(StandardCharsets.UTF_8);
366        Exception failure = null;
367        boolean contentStored = false;
368        try {
369            m_s3Client.validateBucketAccess();
370            m_s3Client.putObject(key, content);
371            contentStored = true;
372            byte[] readContent = m_s3Client.getObject(key);
373            if (!Arrays.equals(content, readContent)) {
374                throw new IllegalStateException(
375                    Messages.get().getBundle().key(
376                        Messages.ERR_STORAGE_S3_READ_WRITE_1,
377                        m_configuration.getBucketName()) + " " + getDiagnosticContext(key));
378            }
379        } catch (Exception e) {
380            failure = e;
381            throw e;
382        } finally {
383            if (contentStored) {
384                try {
385                    m_s3Client.deleteObject(key);
386                } catch (Exception e) {
387                    if (failure != null) {
388                        failure.addSuppressed(e);
389                    } else {
390                        throw e;
391                    }
392                }
393            }
394        }
395    }
396
397    /**
398     * @see org.opencms.db.storage.I_CmsEnumerableStorage#visitContentHashes(org.opencms.db.CmsDbContext, org.opencms.db.storage.I_CmsEnumerableStorage.I_CmsContentHashVisitor)
399     */
400    @Override
401    public void visitContentHashes(CmsDbContext dbc, I_CmsContentHashVisitor visitor) throws Exception {
402
403        m_s3Client.visitObjectKeys(key -> {
404            String hash = getHashFromObjectKey(key);
405            if (hash != null) {
406                visitor.visit(hash);
407            }
408        });
409    }
410
411    /**
412     * Returns the configured single API call attempt timeout.
413     *
414     * @return the timeout in milliseconds
415     */
416    int getApiCallAttemptTimeout() {
417
418        return m_configuration.getApiCallAttemptTimeout();
419    }
420
421    /**
422     * Returns the configured complete API call timeout.
423     *
424     * @return the timeout in milliseconds
425     */
426    int getApiCallTimeout() {
427
428        return m_configuration.getApiCallTimeout();
429    }
430
431    /**
432     * Returns the configured connection acquisition timeout.<p>
433     *
434     * @return the timeout in milliseconds
435     */
436    int getConnectionAcquisitionTimeout() {
437
438        return m_configuration.getConnectionAcquisitionTimeout();
439    }
440
441    /**
442     * Returns the configured connection timeout.
443     *
444     * @return the timeout in milliseconds
445     */
446    int getConnectionTimeout() {
447
448        return m_configuration.getConnectionTimeout();
449    }
450
451    /**
452     * Returns the configured maximum number of pooled connections.<p>
453     *
454     * @return the maximum number of pooled connections
455     */
456    int getMaxConnections() {
457
458        return m_configuration.getMaxConnections();
459    }
460
461    /**
462     * Returns the configured maximum number of retries.
463     *
464     * @return the maximum number of retries
465     */
466    int getMaxRetries() {
467
468        return m_configuration.getMaxRetries();
469    }
470
471    /**
472     * Returns the configured AWS region.
473     *
474     * @return the AWS region
475     */
476    String getRegion() {
477
478        return m_configuration.getRegion();
479    }
480
481    /**
482     * Returns the configured socket timeout.
483     *
484     * @return the timeout in milliseconds
485     */
486    int getSocketTimeout() {
487
488        return m_configuration.getSocketTimeout();
489    }
490
491    /**
492     * Gets diagnostic context for health check failures.<p>
493     *
494     * @param key the S3 key
495     * @return the diagnostic context
496     */
497    private String getDiagnosticContext(String key) {
498
499        return "storage="
500            + getStorageIdentifier()
501            + ", endpoint="
502            + m_configuration.getEndpoint()
503            + ", bucket="
504            + m_configuration.getBucketName()
505            + ", key="
506            + key
507            + ", region="
508            + m_configuration.getRegion()
509            + ", pathStyle="
510            + m_configuration.isPathStyle();
511    }
512
513    /**
514     * Extracts a valid content hash from an object key.<p>
515     *
516     * @param key the object key
517     * @return the content hash, or null if the key is not a content object
518     */
519    private String getHashFromObjectKey(String key) {
520
521        if (key == null) {
522            return null;
523        }
524        int slashPos = key.lastIndexOf('/');
525        String candidate = slashPos >= 0 ? key.substring(slashPos + 1) : key;
526        try {
527            String hash = validateHash(candidate);
528            String expectedKey = getHashedPath(hash);
529            return expectedKey.equals(key) ? hash : null;
530        } catch (IllegalArgumentException e) {
531            return null;
532        }
533    }
534
535}