Source code

001/*
002 * This library is part of OpenCms -
003 * the Open Source Content Management System
004 *
005 * Copyright (c) Alkacon Software GmbH & Co. KG (https://www.alkacon.com)
006 *
007 * This library is free software; you can redistribute it and/or
008 * modify it under the terms of the GNU Lesser General Public
009 * License as published by the Free Software Foundation; either
010 * version 2.1 of the License, or (at your option) any later version.
011 *
012 * This library is distributed in the hope that it will be useful,
013 * but WITHOUT ANY WARRANTY; without even the implied warranty of
014 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
015 * Lesser General Public License for more details.
016 *
017 * For further information about Alkacon Software, please see the
018 * company website: https://www.alkacon.com
019 *
020 * For further information about OpenCms, please see the
021 * project website: https://www.opencms.org
022 *
023 * You should have received a copy of the GNU Lesser General Public
024 * License along with this library; if not, write to the Free Software
025 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
026 */
027
028package org.opencms.db.storage;
029
030import org.opencms.db.CmsDbContext;
031import org.opencms.file.I_CmsFileContentStreamHandler;
032
033import java.io.IOException;
034import java.io.InputStream;
035import java.io.OutputStream;
036import java.nio.ByteBuffer;
037import java.nio.channels.SeekableByteChannel;
038import java.nio.charset.StandardCharsets;
039import java.nio.file.DirectoryNotEmptyException;
040import java.nio.file.FileAlreadyExistsException;
041import java.nio.file.Files;
042import java.nio.file.LinkOption;
043import java.nio.file.NoSuchFileException;
044import java.nio.file.Path;
045import java.nio.file.Paths;
046import java.nio.file.StandardCopyOption;
047import java.nio.file.StandardOpenOption;
048import java.util.Arrays;
049import java.util.stream.Stream;
050
051/**
052 * File system backend for the OpenCms deduplicating media storage engine (DMSE).<p>
053 *
054 * This implementation stores binary content as files in the local file system.
055 * It uses a 3-level directory structure based on the content hash to avoid
056 * having too many files in a single directory.<p>
057 *
058 * @see org.opencms.db.storage.CmsStorageManager
059 */
060public class CmsFsStorage extends A_CmsStorage implements I_CmsEnumerableStorage, I_CmsStorageDelivery {
061
062    /** The type name of the storage implementation. */
063    public static final String STORAGE_TYPE = "fs";
064
065    /** The base path of the storage repository. */
066    private final Path m_repository;
067
068    /** The configured backend id. */
069    private final String m_name;
070
071    /**
072     * Creates a new file system storage.<p>
073     *
074     * @param name the configured backend id used as storage identifier
075     * @param basePath the base path for the storage repository
076     */
077    public CmsFsStorage(String name, String basePath) {
078
079        m_name = name;
080        m_repository = Paths.get(basePath).toAbsolutePath().normalize();
081    }
082
083    /**
084     * @see org.opencms.db.storage.I_CmsStorage#deleteContent(org.opencms.db.CmsDbContext, java.lang.String)
085     */
086    @Override
087    public void deleteContent(CmsDbContext dbc, String hash) throws Exception {
088
089        Path file = getContentPath(hash);
090        if (Files.exists(file, LinkOption.NOFOLLOW_LINKS) && !Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS)) {
091            throw new IOException(Messages.get().getBundle().key(Messages.ERR_STORAGE_PATH_NOT_REGULAR_FILE_1, file));
092        }
093        if (Files.deleteIfExists(file)) {
094            deleteEmptyParents(file.getParent());
095        }
096    }
097
098    /**
099     * @see org.opencms.db.storage.I_CmsStorage#getStorageIdentifier()
100     */
101    @Override
102    public String getStorageIdentifier() {
103
104        return m_name;
105    }
106
107    /**
108     * @see org.opencms.db.storage.I_CmsStorage#loadContent(org.opencms.db.CmsDbContext, java.lang.String)
109     */
110    @Override
111    public byte[] loadContent(CmsDbContext dbc, String hash) throws Exception {
112
113        return Files.readAllBytes(getExistingRegularContentPath(hash));
114    }
115
116    /**
117     * @see I_CmsStorage#loadContentFrom(CmsDbContext, String, I_CmsFileContentStreamHandler)
118     */
119    @Override
120    public void loadContentFrom(CmsDbContext dbc, String hash, I_CmsFileContentStreamHandler handler) throws Exception {
121
122        try (InputStream in = Files.newInputStream(getExistingRegularContentPath(hash))) {
123            handler.read(in);
124        }
125    }
126
127    /**
128     * @see I_CmsStorage#loadContentTo(CmsDbContext, String, OutputStream)
129     */
130    @Override
131    public void loadContentTo(CmsDbContext dbc, String hash, OutputStream out) throws Exception {
132
133        Files.copy(getExistingRegularContentPath(hash), out);
134    }
135
136    /**
137     * @see org.opencms.db.storage.I_CmsStorage#storeContent(org.opencms.db.CmsDbContext, java.lang.String, byte[])
138     */
139    @Override
140    public void storeContent(CmsDbContext dbc, String hash, byte[] content) throws Exception {
141
142        if (content == null) {
143            throw new IllegalArgumentException(Messages.get().getBundle().key(Messages.ERR_STORAGE_CONTENT_NULL_0));
144        }
145        Path file = getContentPath(hash);
146        if (Files.exists(file, LinkOption.NOFOLLOW_LINKS)) {
147            if (!Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS)) {
148                throw new IOException(
149                    Messages.get().getBundle().key(Messages.ERR_STORAGE_PATH_NOT_REGULAR_FILE_1, file));
150            }
151            return;
152        }
153        Path parent = file.getParent();
154        Files.createDirectories(parent);
155        Path tempFile = Files.createTempFile(parent, hash, ".tmp");
156        try {
157            Files.write(tempFile, content);
158            Files.move(tempFile, file, StandardCopyOption.ATOMIC_MOVE);
159        } catch (FileAlreadyExistsException e) {
160            Files.deleteIfExists(tempFile);
161            if (!Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS)) {
162                throw new IOException(
163                    Messages.get().getBundle().key(Messages.ERR_STORAGE_PATH_NOT_REGULAR_FILE_1, file),
164                    e);
165            }
166        } catch (Exception e) {
167            Files.deleteIfExists(tempFile);
168            throw e;
169        }
170    }
171
172    /**
173     * @see org.opencms.db.storage.I_CmsStorageDelivery#streamRangeTo(org.opencms.db.CmsDbContext, java.lang.String, long, long, java.io.OutputStream)
174     */
175    @Override
176    public void streamRangeTo(CmsDbContext dbc, String hash, long start, long length, OutputStream out)
177    throws Exception {
178
179        try (SeekableByteChannel channel = Files.newByteChannel(
180            getExistingRegularContentPath(hash),
181            StandardOpenOption.READ)) {
182            channel.position(start);
183            byte[] buffer = new byte[8192];
184            long remaining = length;
185            while (remaining > 0) {
186                ByteBuffer byteBuffer = ByteBuffer.wrap(buffer, 0, (int)Math.min(buffer.length, remaining));
187                int read = channel.read(byteBuffer);
188                if (read < 0) {
189                    return;
190                }
191                out.write(buffer, 0, read);
192                remaining -= read;
193            }
194        }
195    }
196
197    /**
198     * @see org.opencms.db.storage.I_CmsStorageDelivery#streamTo(org.opencms.db.CmsDbContext, java.lang.String, java.io.OutputStream)
199     */
200    @Override
201    public void streamTo(CmsDbContext dbc, String hash, OutputStream out) throws Exception {
202
203        loadContentTo(dbc, hash, out);
204    }
205
206    /**
207     * @see org.opencms.db.storage.I_CmsStorageDelivery#supportsRangeDelivery()
208     */
209    @Override
210    public boolean supportsRangeDelivery() {
211
212        return true;
213    }
214
215    /**
216     * @see org.opencms.db.storage.I_CmsStorage#validateAvailable(org.opencms.db.CmsDbContext)
217     */
218    @Override
219    public void validateAvailable(CmsDbContext dbc) throws Exception {
220
221        if (!Files.exists(m_repository, LinkOption.NOFOLLOW_LINKS)) {
222            throw new IOException(
223                Messages.get().getBundle().key(Messages.ERR_STORAGE_REPOSITORY_MISSING_1, m_repository));
224        }
225        if (!Files.isDirectory(m_repository, LinkOption.NOFOLLOW_LINKS)) {
226            throw new IOException(
227                Messages.get().getBundle().key(Messages.ERR_STORAGE_REPOSITORY_NOT_DIRECTORY_1, m_repository));
228        }
229        byte[] content = "OpenCms storage health check".getBytes(StandardCharsets.UTF_8);
230        Path testFile = Files.createTempFile(m_repository, ".opencms-healthcheck-", ".tmp");
231        try {
232            Files.write(testFile, content);
233            byte[] readContent = Files.readAllBytes(testFile);
234            if (!Arrays.equals(content, readContent)) {
235                throw new IOException(
236                    Messages.get().getBundle().key(Messages.ERR_STORAGE_REPOSITORY_READ_WRITE_1, m_repository));
237            }
238        } finally {
239            Files.deleteIfExists(testFile);
240        }
241    }
242
243    /**
244     * @see org.opencms.db.storage.I_CmsEnumerableStorage#visitContentHashes(org.opencms.db.CmsDbContext, org.opencms.db.storage.I_CmsEnumerableStorage.I_CmsContentHashVisitor)
245     */
246    @Override
247    public void visitContentHashes(CmsDbContext dbc, I_CmsContentHashVisitor visitor) throws Exception {
248
249        try (Stream<Path> files = Files.walk(m_repository)) {
250            for (Path file : (Iterable<Path>)files::iterator) {
251                if (!Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS)) {
252                    continue;
253                }
254                String name = file.getFileName().toString();
255                if (isHash(name)) {
256                    visitor.visit(name.toLowerCase());
257                }
258            }
259        }
260    }
261
262    /**
263     * Deletes empty parent folders up to, but not including, the repository root.<p>
264     *
265     * @param start the folder to start with
266     * @throws IOException if deleting an empty folder fails
267     */
268    private void deleteEmptyParents(Path start) throws IOException {
269
270        Path current = start;
271        while ((current != null) && !current.equals(m_repository)) {
272            try {
273                Files.delete(current);
274            } catch (DirectoryNotEmptyException e) {
275                return;
276            } catch (NoSuchFileException e) {
277                return;
278            }
279            current = current.getParent();
280        }
281    }
282
283    /**
284     * Resolves the storage path for the given content hash.<p>
285     *
286     * @param hash the content hash
287     * @return the storage path
288     */
289    private Path getContentPath(String hash) {
290
291        String normalizedHash = validateHash(hash);
292        Path result = m_repository.resolve(normalizedHash.substring(0, 2)).resolve(
293            normalizedHash.substring(2, 4)).resolve(normalizedHash.substring(4, 6)).resolve(normalizedHash).normalize();
294        if (!result.startsWith(m_repository)) {
295            throw new IllegalArgumentException(
296                Messages.get().getBundle().key(Messages.ERR_STORAGE_HASH_OUTSIDE_REPOSITORY_0));
297        }
298        return result;
299    }
300
301    /**
302     * Returns the existing regular content file for the given hash.<p>
303     *
304     * @param hash the SHA-512 content hash
305     * @return the content file path
306     * @throws IOException if the content path is invalid
307     * @throws CmsStorageBlobNotFoundException if the content file does not exist
308     */
309    private Path getExistingRegularContentPath(String hash) throws IOException, CmsStorageBlobNotFoundException {
310
311        Path file = getContentPath(hash);
312        if (!Files.exists(file, LinkOption.NOFOLLOW_LINKS)) {
313            throw new CmsStorageBlobNotFoundException(
314                Messages.get().getBundle().key(Messages.ERR_STORAGE_BLOB_MISSING_2, getStorageIdentifier(), hash));
315        }
316        if (!Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS)) {
317            throw new IOException(Messages.get().getBundle().key(Messages.ERR_STORAGE_PATH_NOT_REGULAR_FILE_1, file));
318        }
319        return file;
320    }
321
322    /**
323     * Checks if the given string looks like a SHA-512 hash.<p>
324     *
325     * @param value the value
326     * @return true if the value looks like a hash
327     */
328    private boolean isHash(String value) {
329
330        if ((value == null) || (value.length() != 128)) {
331            return false;
332        }
333        for (int i = 0; i < value.length(); i++) {
334            char c = value.charAt(i);
335            if (!(((c >= '0') && (c <= '9')) || ((c >= 'a') && (c <= 'f')) || ((c >= 'A') && (c <= 'F')))) {
336                return false;
337            }
338        }
339        return true;
340    }
341
342}