001/*
002 * This library is part of OpenCms -
003 * the Open Source Content Management System
004 *
005 * Copyright (c) Alkacon Software GmbH & Co. KG (https://www.alkacon.com)
006 *
007 * This library is free software; you can redistribute it and/or
008 * modify it under the terms of the GNU Lesser General Public
009 * License as published by the Free Software Foundation; either
010 * version 2.1 of the License, or (at your option) any later version.
011 *
012 * This library is distributed in the hope that it will be useful,
013 * but WITHOUT ANY WARRANTY; without even the implied warranty of
014 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
015 * Lesser General Public License for more details.
016 *
017 * For further information about Alkacon Software GmbH & Co. KG, please see the
018 * company website: https://www.alkacon.com
019 *
020 * For further information about OpenCms, please see the
021 * project website: https://www.opencms.org
022 *
023 * You should have received a copy of the GNU Lesser General Public
024 * License along with this library; if not, write to the Free Software
025 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
026 */
027
028package org.opencms.scheduler.jobs;
029
030import org.opencms.configuration.CmsImageCacheConfiguration;
031import org.opencms.configuration.CmsImageCacheConfiguration.RetentionMode;
032import org.opencms.file.CmsObject;
033import org.opencms.loader.CmsImageLoader;
034import org.opencms.loader.CmsS3ImageCache;
035import org.opencms.loader.I_CmsImageCache;
036import org.opencms.loader.imagecache.CmsImageCacheMaintenanceCleaner;
037import org.opencms.loader.imagecache.CmsImageCacheMaintenanceService;
038import org.opencms.main.CmsLog;
039import org.opencms.main.OpenCms;
040import org.opencms.scheduler.I_CmsScheduledJob;
041
042import java.io.File;
043import java.time.Instant;
044import java.util.Map;
045
046import org.apache.commons.logging.Log;
047
048/**
049 * A schedulable OpenCms job that clear the image cache for the scaled images created by the <code>{@link org.opencms.loader.CmsImageLoader}</code>.<p>
050 *
051 * Job parameters:<p>
052 * <dl>
053 * <dt><code>maxage={time in hours}</code></dt>
054 * <dd>Specifies the maximum age (in hours) images can be unused before they are removed from the cache.
055 * Any image in the image cache folder that has a RFS date of last modification older than this time is considered
056 * expired and is therefore deleted. This parameter is only used by the classic RFS image cache. Shared-FS and S3
057 * use the retention and cleanup settings from {@code opencms-system.xml}. Cleanup is skipped when the retention mode
058 * is {@code external}.</dd>
059 * </dl>
060 *
061 * @since 6.2.0
062 */
063public class CmsImageCacheCleanupJob implements I_CmsScheduledJob {
064
065    /** Number of Shared-FS entries passed to one delete request. */
066    private static final int FS_DELETE_BATCH_SIZE = 1000;
067
068    /** Unlock parameter. */
069    public static final String PARAM_MAXAGE = "maxage";
070
071    /** The log object for this class. */
072    private static final Log LOG = CmsLog.getLog(CmsImageCacheCleanupJob.class);
073
074    /**
075     * Removes all expired image cache entries from the RFS cache.<p>
076     *
077     * Empty directories are removed as well.<p>
078     *
079     * @param maxAge the maximum age of the image cache files in hours (or fractions of hours)
080     *
081     * @return the total number of deleted resources
082     */
083    public static int cleanImageCache(float maxAge) {
084
085        String repositoryPath = CmsImageLoader.getImageRepositoryPath();
086        if ((CmsImageLoader.getImageCache() != null) || (repositoryPath == null)) {
087            return 0;
088        }
089        // calculate oldest possible date for the cache files
090        long expireDate = System.currentTimeMillis() - (long)(maxAge * 60f * 60f * 1000f);
091        File basedir = new File(repositoryPath);
092        // perform the cache cleanup
093        return cleanImageCache(expireDate, basedir);
094    }
095
096    /**
097     * Cleans an external image cache according to the central XML configuration.<p>
098     *
099     * @param imageCache the external image cache
100     * @param configuration the image cache configuration
101     * @param now the time used to calculate the retention cutoff
102     * @param listener the optional progress listener
103     * @return the cleanup result
104     * @throws Exception if cleanup fails
105     */
106    static CmsImageCacheMaintenanceCleaner.Result cleanExternalImageCache(
107        I_CmsImageCache imageCache,
108        CmsImageCacheConfiguration configuration,
109        Instant now,
110        CmsImageCacheMaintenanceCleaner.I_ProgressListener listener)
111    throws Exception {
112
113        if ((configuration == null) || !configuration.isConfigured()) {
114            throw new IllegalArgumentException("External image cache cleanup requires an imagecache configuration.");
115        }
116        if (configuration.getRetentionMode() == RetentionMode.external) {
117            return new CmsImageCacheMaintenanceCleaner.Result();
118        }
119        CmsImageCacheMaintenanceService service = CmsImageCacheMaintenanceService.create(imageCache, configuration);
120        int deleteBatchSize = Math.min(FS_DELETE_BATCH_SIZE, configuration.getCleanupMaxDeletesPerRun());
121        if (imageCache instanceof CmsS3ImageCache) {
122            long concurrentBatchSize = (long)configuration.getS3DeleteBatchSize()
123                * configuration.getS3DeleteConcurrency();
124            deleteBatchSize = (int)Math.min(
125                configuration.getCleanupMaxDeletesPerRun(),
126                Math.min(Integer.MAX_VALUE, concurrentBatchSize));
127        }
128        Instant cutoff = now.minus(configuration.getMaxAge());
129        return CmsImageCacheMaintenanceCleaner.delete(
130            service,
131            cutoff,
132            deleteBatchSize,
133            configuration.getCleanupMaxDeletesPerRun(),
134            configuration.getCleanupMaxRuntime(),
135            listener);
136    }
137
138    /**
139     * Removes all expired image cache entries from the given RFS directory, including recursion to subdirectories.<p>
140     *
141     * @param maxAge the maximum age of the image cache files
142     * @param directory the directory to remove the cache files in
143     *
144     * @return the total number of deleted resources
145     */
146    private static int cleanImageCache(long maxAge, File directory) {
147
148        int count = 0;
149        if (directory.canRead() && directory.isDirectory()) {
150            File[] files = directory.listFiles();
151            for (int i = 0; i < files.length; i++) {
152                File f = files[i];
153                if (f.isDirectory()) {
154                    count += cleanImageCache(maxAge, f);
155                }
156                if (f.canWrite()) {
157                    if (f.lastModified() < maxAge) {
158                        try {
159                            f.delete();
160                            count++;
161                        } catch (Exception e) {
162                            LOG.error(
163                                Messages.get().getBundle().key(
164                                    Messages.LOG_IMAGE_CACHE_UNABLE_TO_DELETE_1,
165                                    f.getAbsolutePath()));
166                        }
167                    }
168                }
169            }
170            if (directory.listFiles().length <= 0) {
171                try {
172                    directory.delete();
173                    count++;
174                } catch (Exception e) {
175                    LOG.error(
176                        Messages.get().getBundle().key(
177                            Messages.LOG_IMAGE_CACHE_UNABLE_TO_DELETE_1,
178                            directory.getAbsolutePath()));
179                }
180            }
181        }
182        return count;
183    }
184
185    /**
186     * @see org.opencms.scheduler.I_CmsScheduledJob#launch(CmsObject, Map)
187     */
188    public String launch(CmsObject cms, Map<String, String> parameters) throws Exception {
189
190        if (!CmsImageLoader.isEnabled()) {
191            // scaling functions are not available
192            return Messages.get().getBundle().key(Messages.LOG_IMAGE_SCALING_DISABLED_0);
193        }
194        I_CmsImageCache imageCache = CmsImageLoader.getImageCache();
195        if (imageCache != null) {
196            CmsImageCacheConfiguration configuration = OpenCms.getImageCacheConfiguration();
197            if (configuration.getRetentionMode() == RetentionMode.external) {
198                return Messages.get().getBundle().key(Messages.LOG_IMAGE_CACHE_CLEANUP_EXTERNALLY_MANAGED_0);
199            }
200            CmsImageCacheMaintenanceCleaner.Result result = cleanExternalImageCache(
201                imageCache,
202                configuration,
203                Instant.now(),
204                (progress, batch) -> {
205                    for (Map.Entry<String, Exception> failure : batch.getFailures().entrySet()) {
206                        LOG.error(
207                            Messages.get().getBundle().key(
208                                Messages.LOG_IMAGE_CACHE_UNABLE_TO_DELETE_1,
209                                failure.getKey()),
210                            failure.getValue());
211                    }
212                });
213            return Messages.get().getBundle().key(
214                Messages.LOG_IMAGE_CACHE_CLEANUP_EXTERNAL_RESULT_8,
215                new Object[] {
216                    imageCache instanceof CmsS3ImageCache ? "s3" : "shared-fs",
217                    Long.valueOf(result.getScanned()),
218                    Long.valueOf(result.getMatched()),
219                    Long.valueOf(result.getSucceeded()),
220                    Long.valueOf(result.getSkipped()),
221                    Long.valueOf(result.getFailed()),
222                    Boolean.valueOf(result.isDeleteLimitReached()),
223                    Boolean.valueOf(result.isRuntimeLimitReached())});
224        }
225        if (CmsImageLoader.getImageRepositoryPath() == null) {
226            throw new IllegalStateException("The RFS image cache repository is not initialized.");
227        }
228
229        String maxAgeStr = parameters.get(PARAM_MAXAGE);
230        float maxAge;
231        try {
232            maxAge = Float.parseFloat(maxAgeStr);
233        } catch (NumberFormatException e) {
234            // in case of an error, use maxage of one week
235            maxAge = 24f * 7f;
236            LOG.error(
237                Messages.get().getBundle().key(
238                    Messages.LOG_IMAGE_CACHE_BAD_MAXAGE_2,
239                    maxAgeStr,
240                    Float.valueOf(maxAge)));
241        }
242
243        // now perform the image cache cleanup
244        int count = cleanImageCache(maxAge);
245
246        return Messages.get().getBundle().key(Messages.LOG_IMAGE_CACHE_CLEANUP_COUNT_1, Integer.valueOf(count));
247    }
248}