Source code

001/*
002 * This library is part of OpenCms -
003 * the Open Source Content Management System
004 *
005 * Copyright (c) Alkacon Software GmbH & Co. KG (https://www.alkacon.com)
006 *
007 * This library is free software; you can redistribute it and/or
008 * modify it under the terms of the GNU Lesser General Public
009 * License as published by the Free Software Foundation; either
010 * version 2.1 of the License, or (at your option) any later version.
011 *
012 * This library is distributed in the hope that it will be useful,
013 * but WITHOUT ANY WARRANTY; without even the implied warranty of
014 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
015 * Lesser General Public License for more details.
016 *
017 * For further information about Alkacon Software, please see the
018 * company website: https://www.alkacon.com
019 *
020 * For further information about OpenCms, please see the
021 * project website: https://www.opencms.org
022 *
023 * You should have received a copy of the GNU Lesser General Public
024 * License along with this library; if not, write to the Free Software
025 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
026 */
027
028package org.opencms.ai;
029
030import org.opencms.ade.configuration.CmsADEConfigData;
031import org.opencms.ade.contenteditor.I_CmsContentTranslator;
032import org.opencms.ai.CmsTranslationUtil.FoundOrCreatedValue;
033import org.opencms.configuration.CmsConfigurationException;
034import org.opencms.configuration.CmsParameterConfiguration;
035import org.opencms.file.CmsFile;
036import org.opencms.file.CmsObject;
037import org.opencms.gwt.shared.CmsGwtConstants;
038import org.opencms.i18n.CmsLocaleManager;
039import org.opencms.main.CmsLog;
040import org.opencms.main.OpenCms;
041import org.opencms.util.CmsStringUtil;
042import org.opencms.xml.content.CmsXmlContent;
043import org.opencms.xml.content.I_CmsXmlContentAugmentation;
044import org.opencms.xml.types.CmsXmlHtmlValue;
045import org.opencms.xml.types.I_CmsXmlContentValue;
046
047import java.nio.charset.StandardCharsets;
048import java.util.ArrayList;
049import java.util.Arrays;
050import java.util.Collection;
051import java.util.HashMap;
052import java.util.List;
053import java.util.Locale;
054import java.util.Map;
055import java.util.stream.Collectors;
056
057import org.apache.commons.lang3.LocaleUtils;
058import org.apache.commons.logging.Log;
059
060import com.deepl.api.DeepLClient;
061import com.deepl.api.DeepLClientOptions;
062import com.deepl.api.Language;
063import com.deepl.api.TextResult;
064import com.deepl.api.TextTranslationOptions;
065import com.google.common.collect.ArrayListMultimap;
066import com.google.common.collect.Multimap;
067
068public class CmsDeeplTranslation implements I_CmsContentTranslator {
069
070    /** Key used for looking up the API key in the secret store. */
071    public static final String SECRET_API_KEY = "contenteditor.translation.deepl.apiKey";
072
073    /** The logger instance for this class. */
074    private static final Log LOG = CmsLog.getLog(CmsDeeplTranslation.class);
075
076    /** Configuration parameter for determining a target locale mapping. */
077    protected static final Object PARAM_TARGET_LOCALE_MAPPING = "targetLocaleMapping";
078
079    /** Cached available target languages. */
080    private volatile List<Language> m_targetLanguages;
081
082    /** Cached available source languages. */
083    private volatile List<Language> m_sourceLanguages;
084
085    /** The parameters from the configuration. */
086    private CmsParameterConfiguration m_params = new CmsParameterConfiguration();
087
088    /** The parsed target locale mapping. */
089    private volatile Map<Locale, Locale> m_targetLocaleMapping;
090
091    /**
092     * Tries to find a suitable Language object from a list of DeepL-supported languages that matches a given locale.
093     *
094     * @param locale the locale
095     * @param languages the languages
096     * @return the matching language
097     */
098    public static Language getMatchingLanguage(Locale locale, List<Language> languages) {
099
100        for (Language language : languages) {
101            if (Locale.forLanguageTag(language.getCode()).equals(locale)) {
102                return language;
103            }
104        }
105        // if not found, fall back to just using the language portion
106        for (Language language : languages) {
107            Locale l1 = new Locale(Locale.forLanguageTag(language.getCode()).getLanguage());
108            Locale l2 = new Locale(locale.getLanguage());
109            if (l1.equals(l2)) {
110                return language;
111            }
112        }
113        return null;
114    }
115
116    /**
117     * @see org.opencms.configuration.I_CmsConfigurationParameterHandler#addConfigurationParameter(java.lang.String, java.lang.String)
118     */
119    @Override
120    public void addConfigurationParameter(String paramName, String paramValue) {
121
122        m_params.add(paramName, paramValue);
123
124    }
125
126    /**
127     * @see org.opencms.configuration.I_CmsConfigurationParameterHandler#getConfiguration()
128     */
129    @Override
130    public CmsParameterConfiguration getConfiguration() {
131
132        return m_params;
133    }
134
135    /**
136     * @see org.opencms.ade.contenteditor.I_CmsContentTranslator#getContentAugmentation()
137     */
138    @Override
139    public I_CmsXmlContentAugmentation getContentAugmentation() {
140
141        return new I_CmsXmlContentAugmentation() {
142
143            @Override
144            public void augmentContent(Context context) throws Exception {
145
146                CmsObject cms = context.getCmsObject();
147                final Locale wpLocale = OpenCms.getWorkplaceManager().getWorkplaceLocale(cms);
148                CmsXmlContent content = context.getContent();
149                Locale sourceLocale = context.getLocale();
150                String targetLocaleParam = context.getParameter(CmsGwtConstants.PARAM_TARGET_LOCALE);
151                String apiKey = OpenCms.getSecretStore().getSecret(SECRET_API_KEY);
152                apiKey = apiKey.trim();
153                Locale targetLocale = CmsLocaleManager.getLocale(targetLocaleParam);
154
155                DeepLClientOptions options = new DeepLClientOptions();
156                DeepLClient client = new DeepLClient(apiKey, options);
157
158                String html = CmsTranslationUtil.getWaitMessage(wpLocale);
159                context.progress(html);
160
161                List<Language> sourceLanguages = getSourceLanguages(client);
162                List<Language> targetLanguages = getTargetLanguages(client);
163                Language srcLang = getMatchingLanguage(sourceLocale, sourceLanguages);
164                Language targetLang = getMatchingLanguage(
165                    getTargetLocaleMapping().getOrDefault(targetLocale, targetLocale),
166                    targetLanguages);
167
168                List<I_CmsXmlContentValue> values = CmsTranslationUtil.getValuesToTranslate(
169                    cms,
170                    content,
171                    sourceLocale,
172                    targetLocale);
173
174                // DeepL can translate multiple strings in one API call. But the total request size is limited,
175                // and also all strings in that API call are translated with the same translation options.
176                // So we first group the values to be translated by whether they are HTML values or not
177                // (This is because when we use HTML mode for non-HTML values, it may replace <, > with entities.).
178                // Then we split each of these two groups into smaller batches which can be translated in one API call each.
179
180                Multimap<Boolean, I_CmsXmlContentValue> valuesByType = ArrayListMultimap.create();
181                for (I_CmsXmlContentValue value : values) {
182                    valuesByType.put(value instanceof CmsXmlHtmlValue, value);
183                }
184
185                // Collect *all* translation results in this map
186                Map<String, String> valuesToSet = new HashMap<>();
187
188                for (Boolean isHtml : Arrays.asList(Boolean.FALSE, Boolean.TRUE)) {
189                    TextTranslationOptions translationOptions = new TextTranslationOptions();
190                    if (isHtml) {
191                        translationOptions.setTagHandlingVersion("v2");
192                        translationOptions.setTagHandling("html");
193                    }
194                    Collection<I_CmsXmlContentValue> valuesForCurrentMode = valuesByType.get(isHtml);
195                    if (valuesForCurrentMode.isEmpty()) {
196                        continue;
197                    }
198                    List<List<I_CmsXmlContentValue>> batches = new ArrayList<>();
199
200                    // Max request size for the API is 128K, but this also includes JSON punctuation and additional parameters, so we leave a wide safety margin
201                    final int limit = 15;
202                    int currentBatchSize = 0;
203                    List<I_CmsXmlContentValue> currentBatch = new ArrayList<>();
204                    batches.add(currentBatch);
205                    for (I_CmsXmlContentValue val : valuesForCurrentMode) {
206                        String strValue = getTranslationValue(cms, val);
207                        long byteSize = strValue.getBytes(StandardCharsets.UTF_8).length;
208                        if ((byteSize + currentBatchSize) > limit) {
209                            currentBatch = new ArrayList<>();
210                            batches.add(currentBatch);
211                            currentBatchSize = 0;
212                        }
213                        currentBatch.add(val);
214                        currentBatchSize += byteSize;
215                    }
216
217                    for (List<I_CmsXmlContentValue> batch : batches) {
218                        if (batch.size() == 0) {
219                            continue;
220                        }
221
222                        List<String> inputs = batch.stream().map(val -> getTranslationValue(cms, val)).collect(
223                            Collectors.toList());
224                        List<TextResult> outputs = client.translateText(
225                            inputs,
226                            srcLang,
227                            targetLang,
228                            translationOptions);
229                        for (int i = 0; i < batch.size(); i++) {
230                            valuesToSet.put(batch.get(i).getPath(), outputs.get(i).getText());
231                        }
232                    }
233                }
234                int numTranslatedFields = 0;
235
236                if (!content.hasLocale(targetLocale)) {
237                    content.copyLocale(sourceLocale, targetLocale);
238                    for (Map.Entry<String, String> entry : valuesToSet.entrySet()) {
239                        I_CmsXmlContentValue targetValue = content.getValue(entry.getKey(), targetLocale);
240                        setTranslationValue(cms, targetValue, entry.getValue());
241                        numTranslatedFields += 1;
242                    }
243                } else {
244                    for (Map.Entry<String, String> entry : valuesToSet.entrySet()) {
245                        I_CmsXmlContentValue origValue = content.getValue(entry.getKey(), sourceLocale);
246                        try {
247                            FoundOrCreatedValue val = CmsTranslationUtil.findOrCreateValue(
248                                cms,
249                                content,
250                                targetLocale,
251                                origValue.getPath());
252                            // If the value already existed, we only want to write to it if it's empty.
253                            // But if it was just created, it might have a default value, which we need to overwrite.
254                            if (val.wasCreated()
255                                || CmsStringUtil.isEmptyOrWhitespaceOnly(val.getValue().getStringValue(cms))) {
256                                if (val.getValue() instanceof CmsXmlHtmlValue) {
257                                    // for HTML values, we need to copy the old value first so the link table is filled
258                                    val.getValue().setStringValue(cms, origValue.getStringValue(cms));
259                                }
260                                setTranslationValue(cms, val.getValue(), entry.getValue());
261                                numTranslatedFields += 1;
262                            }
263                        } catch (Exception e) {
264                            LOG.debug(e.getLocalizedMessage(), e);
265                        }
266                    }
267                }
268
269                if (numTranslatedFields > 0) {
270                    context.setResult(content);
271                    context.setHtmlMessage(buildFeedbackHtml(cms, sourceLocale, targetLocale, numTranslatedFields));
272                    context.setNextLocale(targetLocale);
273                } else {
274                    String nothingTranslated = Messages.get().getBundle(wpLocale).key(
275                        Messages.GUI_TRANSLATION_NOTHING_TRANSLATED_0);
276                    context.setHtmlMessage("<p>" + nothingTranslated + "</p>");
277                }
278
279            }
280
281        };
282
283    }
284
285    /**
286     * @see org.opencms.configuration.I_CmsConfigurationParameterHandler#initConfiguration()
287     */
288    @Override
289    public void initConfiguration() throws CmsConfigurationException {
290
291    }
292
293    @Override
294    public void initialize(CmsObject cms) {
295
296    }
297
298    /**
299     * @see org.opencms.ade.contenteditor.I_CmsContentTranslator#isEnabled(org.opencms.file.CmsObject, org.opencms.ade.configuration.CmsADEConfigData, org.opencms.file.CmsFile)
300     */
301    @Override
302    public boolean isEnabled(CmsObject cms, CmsADEConfigData config, CmsFile file) {
303
304        return OpenCms.getSecretStore().getSecret(SECRET_API_KEY) != null;
305
306    }
307
308    /**
309     * Gets (and lazily creates if necessary) the target locale mapping.
310     *
311     * <p>DeepL API only lists en-US / en-GB  for English, and pt-BR / pt-PT for Portuguese as target languages.
312     * Using just en or pt might work, but it's probably a good idea to be specific,
313     * So if just en or pt is specified, we map it to the variant that's more widely used by default.
314     *
315     * <p>This can be overriden by the configuration parameter targetLocaleMapping, which has the form l1:r1|l2:r2|..., where the li is
316     * the locale to map, and the ri is the locale to map to.
317     * @return
318     */
319    protected Map<Locale, Locale> getTargetLocaleMapping() {
320
321        if (m_targetLocaleMapping == null) {
322            Map<Locale, Locale> targetLocaleMapping = new HashMap<>();
323            targetLocaleMapping.put(Locale.ENGLISH, Locale.US);
324            targetLocaleMapping.put(new Locale("pt"), Locale.forLanguageTag("pt-PT"));
325            String localeMappingStr = m_params.get(PARAM_TARGET_LOCALE_MAPPING);
326            if (!CmsStringUtil.isEmptyOrWhitespaceOnly(localeMappingStr)) {
327                Map<String, String> mappings = CmsStringUtil.splitAsMap(localeMappingStr.trim(), "|", ":");
328                for (Map.Entry<String, String> entry : mappings.entrySet()) {
329                    targetLocaleMapping.put(
330                        LocaleUtils.toLocale(entry.getKey()),
331                        LocaleUtils.toLocale(entry.getValue()));
332                }
333            }
334            m_targetLocaleMapping = targetLocaleMapping;
335        }
336
337        return m_targetLocaleMapping;
338    }
339
340    /**
341     * Builds the HTML for the feedback screen.
342     *
343     * @param cms the CMS context
344     * @param sourceLocale the translation source locale
345     * @param targetLocale the translation target locale
346     * @param numSuccessfulFieldUpdates the number of translated fields
347     * @param conflictFields the list of fields with conflicts
348     * @return
349     */
350    private String buildFeedbackHtml(
351        CmsObject cms,
352        Locale sourceLocale,
353        Locale targetLocale,
354        int numSuccessfulFieldUpdates) {
355
356        StringBuilder buffer = new StringBuilder();
357        Locale wpLocale = OpenCms.getWorkplaceManager().getWorkplaceLocale(cms);
358        buffer.append("<p>");
359        buffer.append(
360            Messages.get().getBundle(wpLocale).key(
361                Messages.GUI_TRANSLATION_FEEDBACK_3,
362                numSuccessfulFieldUpdates,
363                sourceLocale.getDisplayName(wpLocale),
364                targetLocale.getDisplayName(wpLocale)));
365        buffer.append("</p>");
366        return buffer.toString();
367    }
368
369    /**
370     * Gets the available source languages from the client, but caches them for later calls.
371     *
372     * @param client the DeepL client
373     * @return the available source languages
374     */
375    private List<Language> getSourceLanguages(DeepLClient client) {
376
377        if (m_sourceLanguages == null) {
378            try {
379                m_sourceLanguages = client.getSourceLanguages();
380            } catch (Exception e) {
381                throw new RuntimeException(e);
382            }
383        }
384        return m_sourceLanguages;
385    }
386
387    /**
388     * Gets the available target languages from the DeepL client, but caches them for later calls
389     *
390     * @param client the DeepL client
391     * @return the available target languages
392     */
393    private List<Language> getTargetLanguages(DeepLClient client) {
394
395        if (m_targetLanguages == null) {
396            try {
397                m_targetLanguages = client.getTargetLanguages();
398            } catch (Exception e) {
399                throw new RuntimeException(e);
400            }
401        }
402        return m_targetLanguages;
403
404    }
405
406    /**
407     * Gets the value to translate, which in the case of HTML values is the value with unexpanded macros, and the normal string value otherwise.
408     *
409     * @param cms the CMS context
410     * @param value an XML content value
411     *
412     * @return the value to translate
413     */
414    private String getTranslationValue(CmsObject cms, I_CmsXmlContentValue value) {
415
416        if (value instanceof CmsXmlHtmlValue) {
417            return ((CmsXmlHtmlValue)value).getRawContent();
418        } else {
419            return value.getStringValue(cms);
420        }
421    }
422
423    /**
424     * Sets the translated value, which in the case of HTML values sets just the content node of the value (with unexpanded macros), and sets the value normally otherwise.
425     *
426     * @param cms the CMS context
427     * @param value the value to modify
428     * @param newValue
429     * @param newValue
430     */
431    private void setTranslationValue(CmsObject cms, I_CmsXmlContentValue value, String newValue) {
432
433        if (value instanceof CmsXmlHtmlValue) {
434            ((CmsXmlHtmlValue)value).setRawContent(newValue);
435        } else {
436            value.setStringValue(cms, newValue);
437        }
438
439    }
440
441}