2021-10-24 19:14:09 +02:00
import io
2021-02-15 20:30:28 +02:00
import os
2023-01-23 23:15:23 +02:00
import re
2023-07-09 12:29:11 +02:00
import tempfile
2021-10-24 18:27:37 +02:00
import uuid
2023-07-09 12:29:11 +02:00
from datetime import datetime
2021-05-18 05:51:33 +02:00
from functools import wraps
2022-02-20 10:09:02 +02:00
from html import unescape
2022-12-26 23:10:43 +02:00
from timeit import default_timer
2021-05-18 05:51:33 +02:00
2021-10-24 18:27:37 +02:00
import argostranslatefiles
from argostranslatefiles import get_supported_formats
2023-07-09 12:29:11 +02:00
from flask import Blueprint , Flask , Response , abort , jsonify , render_template , request , send_file , session , url_for
from flask_babel import Babel
from flask_session import Session
2020-12-20 22:05:22 +02:00
from flask_swagger import swagger
from flask_swagger_ui import get_swaggerui_blueprint
2021-10-24 18:27:37 +02:00
from translatehtml import translate_html
2022-12-26 23:10:43 +02:00
from werkzeug . exceptions import HTTPException
2023-03-09 21:05:09 +02:00
from werkzeug . http import http_date
2023-07-09 12:29:11 +02:00
from werkzeug . utils import secure_filename
2021-05-18 05:51:33 +02:00
2023-07-09 12:29:11 +02:00
from libretranslate import flood , remove_translated_files , scheduler , secret , security , storage
2022-12-30 01:44:53 +02:00
from libretranslate . language import detect_languages , improve_translation_formatting
2023-07-09 12:29:11 +02:00
from libretranslate . locales import (
_ ,
_lazy ,
get_alternate_locale_links ,
get_available_locale_codes ,
get_available_locales ,
gettext_escaped ,
gettext_html ,
lazy_swag ,
)
2022-02-20 10:09:02 +02:00
2022-06-21 20:57:32 +02:00
from . api_keys import Database , RemoteDatabase
2021-10-09 11:44:00 +02:00
from . suggestions import Database as SuggestionsDatabase
2021-02-15 20:30:28 +02:00
2021-10-26 21:32:06 +02:00
def get_version ( ) :
try :
with open ( " VERSION " ) as f :
return f . read ( ) . strip ( )
except :
return " ? "
2021-11-05 15:55:45 +02:00
2021-10-25 11:06:39 +02:00
def get_upload_dir ( ) :
2021-10-25 11:46:49 +02:00
upload_dir = os . path . join ( tempfile . gettempdir ( ) , " libretranslate-files-translate " )
if not os . path . isdir ( upload_dir ) :
os . mkdir ( upload_dir )
return upload_dir
2021-11-05 15:55:45 +02:00
2021-10-26 23:13:51 +02:00
def get_req_api_key ( ) :
if request . is_json :
json = get_json_dict ( request )
ak = json . get ( " api_key " )
else :
ak = request . values . get ( " api_key " )
return ak
2021-10-25 11:46:49 +02:00
2023-03-09 20:59:25 +02:00
def get_req_secret ( ) :
if request . is_json :
json = get_json_dict ( request )
ak = json . get ( " secret " )
else :
ak = request . values . get ( " secret " )
return ak
2021-11-05 15:55:45 +02:00
2021-03-08 00:23:25 +02:00
def get_json_dict ( request ) :
d = request . get_json ( )
if not isinstance ( d , dict ) :
2023-01-04 19:15:18 +02:00
abort ( 400 , description = _ ( " Invalid JSON format " ) )
2021-03-08 00:23:25 +02:00
return d
2021-05-18 05:41:02 +02:00
2020-12-21 18:16:49 +02:00
def get_remote_address ( ) :
if request . headers . getlist ( " X-Forwarded-For " ) :
2021-09-08 21:41:12 +02:00
ip = request . headers . getlist ( " X-Forwarded-For " ) [ 0 ] . split ( " , " ) [ 0 ]
2020-12-21 18:16:49 +02:00
else :
2021-05-18 05:41:02 +02:00
ip = request . remote_addr or " 127.0.0.1 "
2020-12-21 18:16:49 +02:00
return ip
2020-12-20 22:05:22 +02:00
2021-06-03 16:36:25 +02:00
2021-10-24 18:27:37 +02:00
def get_req_limits ( default_limit , api_keys_db , multiplier = 1 ) :
2021-06-03 16:36:25 +02:00
req_limit = default_limit
if api_keys_db :
2021-10-26 23:13:51 +02:00
api_key = get_req_api_key ( )
2021-06-03 16:36:25 +02:00
if api_key :
db_req_limit = api_keys_db . lookup ( api_key )
if db_req_limit is not None :
2021-07-04 18:36:13 +02:00
req_limit = db_req_limit * multiplier
2021-10-24 18:27:37 +02:00
2021-06-03 16:36:25 +02:00
return req_limit
2023-11-10 21:56:52 +02:00
def get_routes_limits ( default_req_limit , hourly_req_limit , daily_req_limit , api_keys_db ) :
2021-02-15 20:30:28 +02:00
if default_req_limit == - 1 :
# TODO: better way?
default_req_limit = 9999999999999
2021-06-03 16:36:25 +02:00
def minute_limits ( ) :
return " %s per minute " % get_req_limits ( default_req_limit , api_keys_db )
2021-02-15 20:30:28 +02:00
2023-11-10 21:56:52 +02:00
def hourly_limits ( ) :
return " %s per hour " % get_req_limits ( hourly_req_limit , api_keys_db , int ( os . environ . get ( " LT_HOURLY_REQ_LIMIT_MULTIPLIER " , 60 ) ) )
2021-06-03 16:36:25 +02:00
def daily_limits ( ) :
2023-11-10 21:56:52 +02:00
return " %s per day " % get_req_limits ( daily_req_limit , api_keys_db , int ( os . environ . get ( " LT_DAILY_REQ_LIMIT_MULTIPLIER " , 1440 ) ) )
2021-03-31 17:57:02 +02:00
2021-06-03 16:36:25 +02:00
res = [ minute_limits ]
2021-05-16 16:57:19 +02:00
2023-11-10 21:56:52 +02:00
if hourly_req_limit > 0 :
res . append ( hourly_limits )
2021-05-16 16:57:19 +02:00
if daily_req_limit > 0 :
2021-06-03 16:36:25 +02:00
res . append ( daily_limits )
2021-05-16 16:57:19 +02:00
return res
2021-02-15 20:30:28 +02:00
2021-05-17 17:41:15 +02:00
2021-02-15 20:30:28 +02:00
def create_app ( args ) :
2022-12-30 01:44:53 +02:00
from libretranslate . init import boot
2021-05-18 05:41:02 +02:00
2023-09-30 00:31:33 +02:00
boot ( args . load_only , args . update_models , args . force_update_models )
2021-01-19 18:51:10 +02:00
2022-12-30 01:44:53 +02:00
from libretranslate . language import load_languages
2021-05-18 05:41:02 +02:00
2023-07-09 12:38:03 +02:00
swagger_url = args . url_prefix + " /docs " # Swagger UI (w/o trailing '/')
api_url = args . url_prefix + " /spec "
2022-12-31 23:44:25 +02:00
2022-12-31 06:23:50 +02:00
bp = Blueprint ( ' Main app ' , __name__ )
2020-12-20 22:05:22 +02:00
2023-03-09 23:09:04 +02:00
storage . setup ( args . shared_storage )
2021-10-25 17:12:09 +02:00
if not args . disable_files_translation :
remove_translated_files . setup ( get_upload_dir ( ) )
2022-03-04 10:23:11 +02:00
languages = load_languages ( )
2022-12-09 23:36:12 +02:00
language_pairs = { }
for lang in languages :
2022-12-10 07:03:21 +02:00
language_pairs [ lang . code ] = sorted ( [ l . to_lang . code for l in lang . translations_from ] )
2021-10-25 17:12:09 +02:00
2021-01-10 10:07:56 +02:00
# Map userdefined frontend languages to argos language object.
2021-02-15 20:30:28 +02:00
if args . frontend_language_source == " auto " :
2021-05-18 05:41:02 +02:00
frontend_argos_language_source = type (
2023-01-04 19:15:18 +02:00
" obj " , ( object , ) , { " code " : " auto " , " name " : _ ( " Auto Detect " ) }
2021-05-18 05:41:02 +02:00
)
2021-01-15 19:01:16 +02:00
else :
2021-05-18 05:41:02 +02:00
frontend_argos_language_source = next (
iter ( [ l for l in languages if l . code == args . frontend_language_source ] ) ,
None ,
)
2023-01-15 19:07:51 +02:00
if frontend_argos_language_source is None :
frontend_argos_language_source = languages [ 0 ]
2021-01-19 18:51:10 +02:00
2023-07-08 04:34:53 +02:00
2023-07-09 12:29:11 +02:00
language_target_fallback = languages [ 1 ] if len ( languages ) > = 2 else languages [ 0 ]
2023-01-15 19:07:51 +02:00
if args . frontend_language_target == " locale " :
def resolve_language_locale ( ) :
loc = get_locale ( )
language_target = next (
iter ( [ l for l in languages if l . code == loc ] ) , None
)
if language_target is None :
language_target = language_target_fallback
return language_target
frontend_argos_language_target = resolve_language_locale
else :
language_target = next (
iter ( [ l for l in languages if l . code == args . frontend_language_target ] ) , None
)
if language_target is None :
language_target = language_target_fallback
frontend_argos_language_target = lambda : language_target
2021-01-19 18:51:10 +02:00
2021-10-24 16:57:45 +02:00
frontend_argos_supported_files_format = [ ]
for file_format in get_supported_formats ( ) :
for ff in file_format . supported_file_extensions :
frontend_argos_supported_files_format . append ( ff )
2021-05-17 17:41:15 +02:00
api_keys_db = None
2021-01-10 10:07:56 +02:00
2023-11-10 21:56:52 +02:00
if args . req_limit > 0 or args . api_keys or args . daily_req_limit > 0 or args . hourly_req_limit > 0 :
2022-06-21 20:57:32 +02:00
api_keys_db = None
if args . api_keys :
2023-07-09 12:29:11 +02:00
api_keys_db = RemoteDatabase ( args . api_keys_remote ) if args . api_keys_remote else Database ( args . api_keys_db_path )
2021-05-17 17:41:15 +02:00
2020-12-20 22:05:22 +02:00
from flask_limiter import Limiter
2021-05-18 05:41:02 +02:00
2020-12-20 22:05:22 +02:00
limiter = Limiter (
key_func = get_remote_address ,
2021-05-18 05:41:02 +02:00
default_limits = get_routes_limits (
2023-11-10 21:56:52 +02:00
args . req_limit , args . hourly_req_limit , args . daily_req_limit , api_keys_db
2021-05-18 05:41:02 +02:00
) ,
2022-10-14 19:27:34 +02:00
storage_uri = args . req_limit_storage ,
2023-11-15 21:34:20 +02:00
default_limits_deduct_when = lambda req : True , # Force cost to be called after the request
default_limits_cost = lambda : getattr ( request , ' req_cost ' , 1 )
2020-12-20 22:05:22 +02:00
)
2021-02-26 16:58:29 +02:00
else :
2021-05-18 05:41:02 +02:00
from . no_limiter import Limiter
limiter = Limiter ( )
2023-03-10 06:07:12 +02:00
if not " gunicorn " in os . environ . get ( " SERVER_SOFTWARE " , " " ) :
# Gunicorn starts the scheduler in the master process
scheduler . setup ( args )
flood . setup ( args )
secret . setup ( args )
2020-12-20 22:05:22 +02:00
2022-12-26 23:10:43 +02:00
measure_request = None
gauge_request = None
if args . metrics :
2023-02-06 01:37:23 +02:00
if os . environ . get ( " PROMETHEUS_MULTIPROC_DIR " ) is None :
default_mp_dir = os . path . abspath ( os . path . join ( " db " , " prometheus " ) )
if not os . path . isdir ( default_mp_dir ) :
os . mkdir ( default_mp_dir )
os . environ [ " PROMETHEUS_MULTIPROC_DIR " ] = default_mp_dir
2023-07-08 04:34:53 +02:00
2023-07-09 12:29:11 +02:00
from prometheus_client import CONTENT_TYPE_LATEST , CollectorRegistry , Gauge , Summary , generate_latest , multiprocess
2022-12-26 23:10:43 +02:00
2022-12-31 06:23:50 +02:00
@bp.route ( " /metrics " )
2022-12-27 03:00:55 +02:00
@limiter.exempt
2022-12-26 23:10:43 +02:00
def prometheus_metrics ( ) :
if args . metrics_auth_token :
authorization = request . headers . get ( ' Authorization ' )
if authorization != " Bearer " + args . metrics_auth_token :
2023-01-05 20:12:35 +02:00
abort ( 401 , description = _ ( " Unauthorized " ) )
2023-07-08 04:34:53 +02:00
2022-12-26 23:10:43 +02:00
registry = CollectorRegistry ( )
multiprocess . MultiProcessCollector ( registry )
return Response ( generate_latest ( registry ) , mimetype = CONTENT_TYPE_LATEST )
2022-12-26 23:49:04 +02:00
measure_request = Summary ( ' libretranslate_http_request_duration_seconds ' , ' Time spent on request ' , [ ' endpoint ' , ' status ' , ' request_ip ' , ' api_key ' ] )
2022-12-26 23:10:43 +02:00
measure_request . labels ( ' /translate ' , 200 , ' 127.0.0.1 ' , ' ' )
2022-12-26 23:49:04 +02:00
gauge_request = Gauge ( ' libretranslate_http_requests_in_flight ' , ' Active requests ' , [ ' endpoint ' , ' request_ip ' , ' api_key ' ] , multiprocess_mode = ' livesum ' )
2022-12-26 23:10:43 +02:00
gauge_request . labels ( ' /translate ' , ' 127.0.0.1 ' , ' ' )
2021-05-17 17:41:15 +02:00
def access_check ( f ) :
@wraps ( f )
def func ( * a , * * kw ) :
2021-11-05 15:55:45 +02:00
ip = get_remote_address ( )
if flood . is_banned ( ip ) :
2023-01-05 20:12:35 +02:00
abort ( 403 , description = _ ( " Too many request limits violations " ) )
2021-05-18 05:41:02 +02:00
2022-02-20 10:06:29 +02:00
if args . api_keys :
2021-10-26 23:13:51 +02:00
ak = get_req_api_key ( )
2023-03-09 20:59:25 +02:00
if ak and api_keys_db . lookup ( ak ) is None :
2022-02-20 10:06:29 +02:00
abort (
403 ,
2023-01-05 20:12:35 +02:00
description = _ ( " Invalid API key " ) ,
2022-02-20 10:06:29 +02:00
)
2023-03-09 20:59:25 +02:00
else :
need_key = False
key_missing = api_keys_db . lookup ( ak ) is None
2023-07-08 04:34:53 +02:00
2023-03-09 20:59:25 +02:00
if ( args . require_api_key_origin
and key_missing
and not re . match ( args . require_api_key_origin , request . headers . get ( " Origin " , " " ) )
) :
need_key = True
2023-07-08 04:34:53 +02:00
2023-03-09 20:59:25 +02:00
if ( args . require_api_key_secret
and key_missing
2023-03-09 23:09:04 +02:00
and not secret . secret_match ( get_req_secret ( ) )
2023-03-09 20:59:25 +02:00
) :
need_key = True
2023-03-09 23:09:04 +02:00
2023-03-09 20:59:25 +02:00
if need_key :
2023-01-05 20:12:35 +02:00
description = _ ( " Please contact the server operator to get an API key " )
2022-06-21 22:39:08 +02:00
if args . get_api_key_link :
2023-01-05 20:12:35 +02:00
description = _ ( " Visit %(url)s to get an API key " , url = args . get_api_key_link )
2021-05-18 05:41:02 +02:00
abort (
2023-03-10 05:00:27 +02:00
400 ,
2022-06-21 22:39:08 +02:00
description = description ,
2021-05-18 05:41:02 +02:00
)
2021-05-17 17:41:15 +02:00
return f ( * a , * * kw )
2023-07-08 04:34:53 +02:00
2022-12-26 23:10:43 +02:00
if args . metrics :
@wraps ( func )
def measure_func ( * a , * * kw ) :
start_t = default_timer ( )
status = 200
ip = get_remote_address ( )
ak = get_req_api_key ( ) or ' '
g = gauge_request . labels ( request . path , ip , ak )
try :
g . inc ( )
return func ( * a , * * kw )
except HTTPException as e :
status = e . code
raise e
finally :
duration = max ( default_timer ( ) - start_t , 0 )
measure_request . labels ( request . path , status , ip , ak ) . observe ( duration )
g . dec ( )
return measure_func
else :
return func
2023-07-08 04:34:53 +02:00
2022-12-31 06:23:50 +02:00
@bp.errorhandler ( 400 )
2020-12-20 22:05:22 +02:00
def invalid_api ( e ) :
return jsonify ( { " error " : str ( e . description ) } ) , 400
2022-12-31 06:23:50 +02:00
@bp.errorhandler ( 500 )
2020-12-20 22:05:22 +02:00
def server_error ( e ) :
return jsonify ( { " error " : str ( e . description ) } ) , 500
2022-12-31 06:23:50 +02:00
@bp.errorhandler ( 429 )
2020-12-20 22:05:22 +02:00
def slow_down_error ( e ) :
2021-05-16 17:50:22 +02:00
flood . report ( get_remote_address ( ) )
2023-01-05 20:12:35 +02:00
return jsonify ( { " error " : _ ( " Slowdown: " ) + " " + str ( e . description ) } ) , 429
2020-12-20 22:05:22 +02:00
2022-12-31 06:23:50 +02:00
@bp.errorhandler ( 403 )
2021-05-16 17:50:22 +02:00
def denied ( e ) :
return jsonify ( { " error " : str ( e . description ) } ) , 403
2022-12-31 06:23:50 +02:00
@bp.route ( " / " )
2021-02-15 20:30:28 +02:00
@limiter.exempt
2020-12-20 22:05:22 +02:00
def index ( ) :
2022-02-07 12:02:32 +02:00
if args . disable_web_ui :
abort ( 404 )
2023-01-06 18:50:51 +02:00
langcode = request . args . get ( ' lang ' )
if langcode and langcode in get_available_locale_codes ( not args . debug ) :
session . update ( preferred_lang = langcode )
2021-05-18 05:41:02 +02:00
return render_template (
" index.html " ,
gaId = args . ga_id ,
frontendTimeout = args . frontend_timeout ,
api_keys = args . api_keys ,
2022-06-21 21:17:42 +02:00
get_api_key_link = args . get_api_key_link ,
2021-05-18 05:41:02 +02:00
web_version = os . environ . get ( " LT_WEB " ) is not None ,
2022-12-31 23:44:25 +02:00
version = get_version ( ) ,
2023-07-09 12:38:03 +02:00
swagger_url = swagger_url ,
2023-01-06 18:50:51 +02:00
available_locales = [ { ' code ' : l [ ' code ' ] , ' name ' : _lazy ( l [ ' name ' ] ) } for l in get_available_locales ( not args . debug ) ] ,
2023-01-05 21:36:50 +02:00
current_locale = get_locale ( ) ,
alternate_locales = get_alternate_locale_links ( )
2021-05-18 05:41:02 +02:00
)
2020-12-20 22:05:22 +02:00
2023-01-06 18:50:51 +02:00
@bp.route ( " /js/app.js " )
2023-01-04 19:15:18 +02:00
@limiter.exempt
def appjs ( ) :
if args . disable_web_ui :
abort ( 404 )
2023-07-08 04:34:53 +02:00
response = Response ( render_template ( " app.js.template " ,
2023-01-06 18:50:51 +02:00
url_prefix = args . url_prefix ,
2023-03-09 20:59:25 +02:00
get_api_key_link = args . get_api_key_link ,
2023-03-09 23:09:04 +02:00
api_secret = secret . get_current_secret ( ) if args . require_api_key_secret else " " ) , content_type = ' application/javascript; charset=utf-8 ' )
2023-07-08 04:34:53 +02:00
2023-03-09 20:59:25 +02:00
if args . require_api_key_secret :
2023-03-09 21:05:09 +02:00
response . headers [ ' Last-Modified ' ] = http_date ( datetime . now ( ) )
2023-03-09 20:59:25 +02:00
response . headers [ ' Cache-Control ' ] = ' no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0 '
response . headers [ ' Pragma ' ] = ' no-cache '
response . headers [ ' Expires ' ] = ' -1 '
2023-07-08 04:34:53 +02:00
2023-03-09 20:59:25 +02:00
return response
2023-01-04 19:15:18 +02:00
2022-12-31 06:23:50 +02:00
@bp.get ( " /languages " )
2021-02-15 20:30:28 +02:00
@limiter.exempt
2020-12-20 22:05:22 +02:00
def langs ( ) :
"""
Retrieve list of supported languages
- - -
tags :
- translate
responses :
200 :
description : List of languages
2021-01-10 10:38:11 +02:00
schema :
id : languages
type : array
items :
type : object
properties :
code :
type : string
description : Language code
name :
type : string
description : Human - readable language name ( in English )
2022-12-10 07:21:38 +02:00
targets :
type : array
items :
type : string
description : Supported target language codes
2020-12-20 22:05:22 +02:00
"""
2023-01-05 20:12:35 +02:00
return jsonify ( [ { " code " : l . code , " name " : _lazy ( l . name ) , " targets " : language_pairs . get ( l . code , [ ] ) } for l in languages ] )
2020-12-20 22:05:22 +02:00
# Add cors
2022-12-31 06:23:50 +02:00
@bp.after_request
2020-12-20 22:05:22 +02:00
def after_request ( response ) :
2021-05-18 05:41:02 +02:00
response . headers . add ( " Access-Control-Allow-Origin " , " * " )
response . headers . add (
" Access-Control-Allow-Headers " , " Authorization, Content-Type "
)
response . headers . add ( " Access-Control-Expose-Headers " , " Authorization " )
response . headers . add ( " Access-Control-Allow-Methods " , " GET, POST " )
response . headers . add ( " Access-Control-Allow-Credentials " , " true " )
response . headers . add ( " Access-Control-Max-Age " , 60 * 60 * 24 * 20 )
2020-12-20 22:05:22 +02:00
return response
2022-12-31 06:23:50 +02:00
@bp.post ( " /translate " )
2021-05-17 17:41:15 +02:00
@access_check
2020-12-20 22:05:22 +02:00
def translate ( ) :
"""
Translate text from a language to another
- - -
tags :
- translate
parameters :
- in : formData
name : q
schema :
2021-01-19 18:51:10 +02:00
oneOf :
- type : string
example : Hello world !
- type : array
example : [ ' Hello world! ' ]
2020-12-20 22:05:22 +02:00
required : true
2021-01-19 18:51:10 +02:00
description : Text ( s ) to translate
2020-12-20 22:05:22 +02:00
- in : formData
name : source
schema :
type : string
example : en
required : true
2021-01-19 18:51:10 +02:00
description : Source language code
2020-12-20 22:05:22 +02:00
- in : formData
name : target
schema :
type : string
example : es
required : true
description : Target language code
2021-09-11 15:08:57 +02:00
- in : formData
name : format
schema :
type : string
2021-09-11 22:02:10 +02:00
enum : [ text , html ]
default : text
example : text
2021-09-11 15:08:57 +02:00
required : false
2021-09-11 22:02:10 +02:00
description : >
Format of source text :
* ` text ` - Plain text
* ` html ` - HTML markup
2021-02-15 20:30:28 +02:00
- in : formData
name : api_key
schema :
type : string
example : xxxxxxxx - xxxx - xxxx - xxxx - xxxxxxxxxxxx
required : false
description : API key
2020-12-20 22:05:22 +02:00
responses :
200 :
description : Translated text
2021-01-10 10:38:11 +02:00
schema :
id : translate
type : object
properties :
translatedText :
2021-01-19 18:51:10 +02:00
oneOf :
- type : string
- type : array
description : Translated text ( s )
2020-12-20 22:05:22 +02:00
400 :
description : Invalid request
2021-01-10 10:38:11 +02:00
schema :
id : error - response
type : object
properties :
error :
type : string
description : Error message
2020-12-20 22:05:22 +02:00
500 :
description : Translation error
2021-01-10 10:38:11 +02:00
schema :
id : error - response
type : object
properties :
error :
type : string
description : Error message
2020-12-20 22:05:22 +02:00
429 :
description : Slow down
2021-01-10 10:38:11 +02:00
schema :
id : error - slow - down
type : object
properties :
error :
type : string
description : Reason for slow down
2021-05-16 17:50:22 +02:00
403 :
description : Banned
schema :
id : error - response
type : object
properties :
error :
type : string
description : Error message
2020-12-20 22:05:22 +02:00
"""
if request . is_json :
2021-03-08 00:23:25 +02:00
json = get_json_dict ( request )
2021-05-18 05:41:02 +02:00
q = json . get ( " q " )
source_lang = json . get ( " source " )
target_lang = json . get ( " target " )
2021-09-11 15:08:57 +02:00
text_format = json . get ( " format " )
2020-12-20 22:05:22 +02:00
else :
q = request . values . get ( " q " )
source_lang = request . values . get ( " source " )
target_lang = request . values . get ( " target " )
2021-09-11 15:08:57 +02:00
text_format = request . values . get ( " format " )
2020-12-20 22:05:22 +02:00
if not q :
2023-01-05 20:12:35 +02:00
abort ( 400 , description = _ ( " Invalid request: missing %(name)s parameter " , name = ' q ' ) )
2020-12-20 22:05:22 +02:00
if not source_lang :
2023-01-05 20:12:35 +02:00
abort ( 400 , description = _ ( " Invalid request: missing %(name)s parameter " , name = ' source ' ) )
2020-12-20 22:05:22 +02:00
if not target_lang :
2023-01-05 20:12:35 +02:00
abort ( 400 , description = _ ( " Invalid request: missing %(name)s parameter " , name = ' target ' ) )
2020-12-20 22:05:22 +02:00
2023-07-08 04:34:53 +02:00
if not request . is_json :
# Normalize line endings to UNIX style (LF) only so we can consistently
# enforce character limits.
# https://www.rfc-editor.org/rfc/rfc2046#section-4.1.1
q = " \n " . join ( q . splitlines ( ) )
2021-01-19 18:51:10 +02:00
batch = isinstance ( q , list )
2021-02-15 20:30:28 +02:00
if batch and args . batch_limit != - 1 :
2021-05-18 05:41:02 +02:00
batch_size = len ( q )
if args . batch_limit < batch_size :
abort (
400 ,
2023-01-05 20:12:35 +02:00
description = _ ( " Invalid request: request ( %(size)s ) exceeds text limit ( %(limit)s ) " , size = batch_size , limit = args . batch_limit ) ,
2021-05-18 05:41:02 +02:00
)
2021-01-19 19:53:53 +02:00
2023-11-15 21:47:44 +02:00
src_texts = q if batch else [ q ]
2023-11-15 21:34:20 +02:00
2021-02-15 20:30:28 +02:00
if args . char_limit != - 1 :
2023-11-15 21:47:44 +02:00
for text in src_texts :
if len ( text ) > args . char_limit :
abort (
400 ,
description = _ ( " Invalid request: request ( %(size)s ) exceeds text limit ( %(limit)s ) " , size = len ( text ) , limit = args . char_limit ) ,
)
2021-01-19 18:51:10 +02:00
2023-11-15 21:47:44 +02:00
if batch :
request . req_cost = max ( 1 , len ( q ) )
2020-12-20 22:05:22 +02:00
2021-05-18 05:41:02 +02:00
if source_lang == " auto " :
2023-11-15 21:47:44 +02:00
candidate_langs = detect_languages ( src_texts )
2023-11-09 04:49:56 +02:00
detected_src_lang = candidate_langs [ 0 ]
2021-08-02 07:06:56 +02:00
else :
2023-11-09 04:49:56 +02:00
detected_src_lang = { " confidence " : 100.0 , " language " : source_lang }
2021-01-19 18:51:10 +02:00
2023-11-09 04:49:56 +02:00
src_lang = next ( iter ( [ l for l in languages if l . code == detected_src_lang [ " language " ] ] ) , None )
2021-10-24 18:27:37 +02:00
2023-11-09 04:49:56 +02:00
if src_lang is None :
abort ( 400 , description = _ ( " %(lang)s is not supported " , lang = source_lang ) )
2021-01-13 16:33:58 +02:00
2020-12-20 22:05:22 +02:00
tgt_lang = next ( iter ( [ l for l in languages if l . code == target_lang ] ) , None )
2021-01-19 18:51:10 +02:00
2020-12-20 22:05:22 +02:00
if tgt_lang is None :
2023-01-05 20:12:35 +02:00
abort ( 400 , description = _ ( " %(lang)s is not supported " , lang = target_lang ) )
2020-12-20 22:05:22 +02:00
2021-09-11 15:08:57 +02:00
if not text_format :
text_format = " text "
if text_format not in [ " text " , " html " ] :
2023-01-05 20:12:35 +02:00
abort ( 400 , description = _ ( " %(format)s format is not supported " , format = text_format ) )
2021-09-11 15:08:57 +02:00
2020-12-20 22:05:22 +02:00
try :
2021-05-18 05:41:02 +02:00
if batch :
2021-08-02 07:06:56 +02:00
results = [ ]
2023-11-09 04:58:59 +02:00
for text in q :
2023-11-09 04:49:56 +02:00
translator = src_lang . get_translation ( tgt_lang )
2022-12-10 07:03:21 +02:00
if translator is None :
2023-11-09 04:49:56 +02:00
abort ( 400 , description = _ ( " %(tname)s ( %(tcode)s ) is not available as a target language from %(sname)s ( %(scode)s ) " , tname = _lazy ( tgt_lang . name ) , tcode = tgt_lang . code , sname = _lazy ( src_lang . name ) , scode = src_lang . code ) )
2022-12-10 07:03:21 +02:00
2021-10-24 18:27:37 +02:00
if text_format == " html " :
translated_text = str ( translate_html ( translator , text ) )
else :
2022-11-16 16:31:25 +02:00
translated_text = improve_translation_formatting ( text , translator . translate ( text ) )
2021-09-11 15:08:57 +02:00
2022-02-06 15:54:52 +02:00
results . append ( unescape ( translated_text ) )
2022-03-25 08:19:56 +02:00
if source_lang == " auto " :
return jsonify (
{
" translatedText " : results ,
2023-11-09 04:49:56 +02:00
" detectedLanguage " : [ detected_src_lang ] * len ( q )
2022-03-25 08:19:56 +02:00
}
)
else :
return jsonify (
2022-12-09 21:35:39 +02:00
{
2022-03-25 08:19:56 +02:00
" translatedText " : results
2022-12-09 21:35:39 +02:00
}
2022-03-25 08:19:56 +02:00
)
2021-05-18 05:41:02 +02:00
else :
2023-11-09 04:49:56 +02:00
translator = src_lang . get_translation ( tgt_lang )
2022-12-10 07:03:21 +02:00
if translator is None :
2023-11-09 04:49:56 +02:00
abort ( 400 , description = _ ( " %(tname)s ( %(tcode)s ) is not available as a target language from %(sname)s ( %(scode)s ) " , tname = _lazy ( tgt_lang . name ) , tcode = tgt_lang . code , sname = _lazy ( src_lang . name ) , scode = src_lang . code ) )
2021-09-11 15:08:57 +02:00
if text_format == " html " :
2021-09-24 17:30:19 +02:00
translated_text = str ( translate_html ( translator , q ) )
2021-09-11 15:08:57 +02:00
else :
2022-11-16 16:31:25 +02:00
translated_text = improve_translation_formatting ( q , translator . translate ( q ) )
2022-03-25 08:19:56 +02:00
if source_lang == " auto " :
return jsonify (
{
" translatedText " : unescape ( translated_text ) ,
2023-11-09 04:49:56 +02:00
" detectedLanguage " : detected_src_lang
2022-03-25 08:19:56 +02:00
}
)
else :
return jsonify (
{
" translatedText " : unescape ( translated_text )
}
)
2020-12-20 22:05:22 +02:00
except Exception as e :
2023-10-30 19:09:39 +02:00
raise e
2023-01-05 20:12:35 +02:00
abort ( 500 , description = _ ( " Cannot translate text: %(text)s " , text = str ( e ) ) )
2020-12-20 22:05:22 +02:00
2022-12-31 06:23:50 +02:00
@bp.post ( " /translate_file " )
2021-10-24 18:27:37 +02:00
@access_check
def translate_file ( ) :
"""
Translate file from a language to another
- - -
tags :
- translate
consumes :
- multipart / form - data
parameters :
- in : formData
name : file
type : file
required : true
2021-10-26 22:04:50 +02:00
description : File to translate
2021-10-24 18:27:37 +02:00
- in : formData
name : source
schema :
type : string
example : en
required : true
description : Source language code
- in : formData
name : target
schema :
type : string
example : es
required : true
description : Target language code
- in : formData
name : api_key
schema :
type : string
example : xxxxxxxx - xxxx - xxxx - xxxx - xxxxxxxxxxxx
required : false
description : API key
responses :
200 :
description : Translated file
schema :
2022-10-18 17:41:36 +02:00
id : translate - file
2021-10-24 18:27:37 +02:00
type : object
properties :
2021-10-25 10:50:55 +02:00
translatedFileUrl :
type : string
description : Translated file url
2021-10-24 18:27:37 +02:00
400 :
description : Invalid request
schema :
id : error - response
type : object
properties :
error :
type : string
description : Error message
500 :
description : Translation error
schema :
id : error - response
type : object
properties :
error :
type : string
description : Error message
429 :
description : Slow down
schema :
id : error - slow - down
type : object
properties :
error :
type : string
description : Reason for slow down
403 :
description : Banned
schema :
id : error - response
type : object
properties :
error :
type : string
description : Error message
"""
2021-10-25 17:09:23 +02:00
if args . disable_files_translation :
2023-01-05 20:12:35 +02:00
abort ( 403 , description = _ ( " Files translation are disabled on this server. " ) )
2021-10-25 17:09:23 +02:00
2021-10-24 19:14:09 +02:00
source_lang = request . form . get ( " source " )
target_lang = request . form . get ( " target " )
2021-10-24 18:27:37 +02:00
file = request . files [ ' file ' ]
if not file :
2023-01-05 20:12:35 +02:00
abort ( 400 , description = _ ( " Invalid request: missing %(name)s parameter " , name = ' file ' ) )
2021-10-24 18:27:37 +02:00
if not source_lang :
2023-01-05 20:12:35 +02:00
abort ( 400 , description = _ ( " Invalid request: missing %(name)s parameter " , name = ' source ' ) )
2021-10-24 18:27:37 +02:00
if not target_lang :
2023-01-05 20:12:35 +02:00
abort ( 400 , description = _ ( " Invalid request: missing %(name)s parameter " , name = ' target ' ) )
2021-10-24 18:27:37 +02:00
if file . filename == ' ' :
2023-01-05 20:12:35 +02:00
abort ( 400 , description = _ ( " Invalid request: empty file " ) )
2021-10-24 18:27:37 +02:00
if os . path . splitext ( file . filename ) [ 1 ] not in frontend_argos_supported_files_format :
2023-01-05 20:12:35 +02:00
abort ( 400 , description = _ ( " Invalid request: file format not supported " ) )
2021-10-24 18:27:37 +02:00
2023-11-09 04:55:55 +02:00
src_lang = next ( iter ( [ l for l in languages if l . code == source_lang ] ) , None )
2021-10-24 18:27:37 +02:00
2023-11-09 04:55:55 +02:00
if src_lang is None :
abort ( 400 , description = _ ( " %(lang)s is not supported " , lang = source_lang ) )
2021-10-24 18:27:37 +02:00
tgt_lang = next ( iter ( [ l for l in languages if l . code == target_lang ] ) , None )
if tgt_lang is None :
2023-01-05 20:12:35 +02:00
abort ( 400 , description = _ ( " %(lang)s is not supported " , lang = target_lang ) )
2021-10-24 18:27:37 +02:00
try :
filename = str ( uuid . uuid4 ( ) ) + ' . ' + secure_filename ( file . filename )
2021-10-25 11:06:39 +02:00
filepath = os . path . join ( get_upload_dir ( ) , filename )
2021-10-24 18:27:37 +02:00
file . save ( filepath )
2023-11-15 21:34:20 +02:00
# Not an exact science: take the number of bytes and divide by
# the character limit. Assuming a plain text file, this will
# set the cost of the request to N = bytes / char_limit, which is
# roughly equivalent to a batch process of N batches assuming
# each batch uses all available limits
if args . char_limit != - 1 :
request . req_cost = max ( 1 , int ( os . path . getsize ( filepath ) / args . char_limit ) )
2023-11-09 04:55:55 +02:00
translated_file_path = argostranslatefiles . translate_file ( src_lang . get_translation ( tgt_lang ) , filepath )
2021-10-24 18:38:35 +02:00
translated_filename = os . path . basename ( translated_file_path )
2021-10-25 10:56:17 +02:00
2021-10-24 18:27:37 +02:00
return jsonify (
{
2023-01-03 14:57:29 +02:00
" translatedFileUrl " : url_for ( ' Main app.download_file ' , filename = translated_filename , _external = True )
2021-10-24 18:27:37 +02:00
}
)
except Exception as e :
abort ( 500 , description = e )
2022-12-31 06:23:50 +02:00
@bp.get ( " /download_file/<string:filename> " )
2021-10-24 18:44:35 +02:00
def download_file ( filename : str ) :
2021-10-24 18:38:35 +02:00
"""
Download a translated file
"""
2021-10-25 17:09:23 +02:00
if args . disable_files_translation :
2023-01-05 20:12:35 +02:00
abort ( 400 , description = _ ( " Files translation are disabled on this server. " ) )
2021-11-05 15:55:45 +02:00
2021-10-25 11:06:39 +02:00
filepath = os . path . join ( get_upload_dir ( ) , filename )
2021-10-26 21:41:14 +02:00
try :
checked_filepath = security . path_traversal_check ( filepath , get_upload_dir ( ) )
if os . path . isfile ( checked_filepath ) :
filepath = checked_filepath
2023-07-09 12:38:03 +02:00
except security . SuspiciousFileOperationError :
2023-01-05 20:12:35 +02:00
abort ( 400 , description = _ ( " Invalid filename " ) )
2021-10-24 19:14:09 +02:00
return_data = io . BytesIO ( )
with open ( filepath , ' rb ' ) as fo :
return_data . write ( fo . read ( ) )
return_data . seek ( 0 )
2021-10-25 12:05:39 +02:00
download_filename = filename . split ( ' . ' )
download_filename . pop ( 0 )
download_filename = ' . ' . join ( download_filename )
2021-10-24 18:44:35 +02:00
2022-04-07 15:56:57 +02:00
return send_file ( return_data , as_attachment = True , download_name = download_filename )
2021-10-24 18:38:35 +02:00
2022-12-31 06:23:50 +02:00
@bp.post ( " /detect " )
2021-05-17 17:41:15 +02:00
@access_check
2021-02-10 17:51:17 +02:00
def detect ( ) :
"""
Detect the language of a single text
- - -
tags :
- translate
parameters :
- in : formData
name : q
schema :
type : string
example : Hello world !
required : true
description : Text to detect
2021-02-15 20:30:28 +02:00
- in : formData
name : api_key
schema :
type : string
example : xxxxxxxx - xxxx - xxxx - xxxx - xxxxxxxxxxxx
required : false
description : API key
2021-02-10 17:51:17 +02:00
responses :
200 :
description : Detections
schema :
id : detections
type : array
items :
type : object
properties :
confidence :
type : number
format : float
minimum : 0
maximum : 1
description : Confidence value
example : 0.6
language :
type : string
description : Language code
example : en
400 :
description : Invalid request
schema :
id : error - response
type : object
properties :
error :
type : string
2021-05-18 05:41:02 +02:00
description : Error message
2021-02-10 17:51:17 +02:00
500 :
description : Detection error
schema :
id : error - response
type : object
properties :
error :
type : string
description : Error message
429 :
description : Slow down
schema :
id : error - slow - down
type : object
properties :
error :
type : string
description : Reason for slow down
2021-05-16 17:50:22 +02:00
403 :
description : Banned
schema :
id : error - response
type : object
properties :
error :
type : string
description : Error message
2021-02-10 17:51:17 +02:00
"""
if request . is_json :
2021-03-08 00:23:25 +02:00
json = get_json_dict ( request )
2021-05-18 05:41:02 +02:00
q = json . get ( " q " )
2021-02-10 17:51:17 +02:00
else :
q = request . values . get ( " q " )
if not q :
2023-01-05 20:12:35 +02:00
abort ( 400 , description = _ ( " Invalid request: missing %(name)s parameter " , name = ' q ' ) )
2021-02-10 17:51:17 +02:00
2021-03-11 11:01:12 +02:00
return jsonify ( detect_languages ( q ) )
2021-02-10 17:51:17 +02:00
2022-12-31 06:23:50 +02:00
@bp.route ( " /frontend/settings " )
2021-02-15 20:30:28 +02:00
@limiter.exempt
2021-01-10 10:07:56 +02:00
def frontend_settings ( ) :
"""
Retrieve frontend specific settings
- - -
tags :
- frontend
responses :
200 :
description : frontend settings
schema :
id : frontend - settings
type : object
properties :
2021-01-10 11:24:42 +02:00
charLimit :
type : integer
description : Character input limit for this language ( - 1 indicates no limit )
2021-01-28 18:16:55 +02:00
frontendTimeout :
type : integer
description : Frontend translation timeout
2021-11-03 16:39:59 +02:00
apiKeys :
type : boolean
description : Whether the API key database is enabled .
2021-10-30 17:31:50 +02:00
keyRequired :
type : boolean
description : Whether an API key is required .
2021-10-09 16:04:16 +02:00
suggestions :
type : boolean
description : Whether submitting suggestions is enabled .
2021-10-24 16:57:45 +02:00
supportedFilesFormat :
type : array
items :
type : string
description : Supported files format
2021-01-10 10:07:56 +02:00
language :
type : object
properties :
source :
type : object
properties :
code :
type : string
description : Language code
name :
type : string
description : Human - readable language name ( in English )
target :
type : object
properties :
code :
type : string
description : Language code
name :
type : string
description : Human - readable language name ( in English )
"""
2023-01-15 19:07:51 +02:00
target_lang = frontend_argos_language_target ( )
2021-05-18 05:41:02 +02:00
return jsonify (
{
" charLimit " : args . char_limit ,
" frontendTimeout " : args . frontend_timeout ,
2021-11-03 16:39:59 +02:00
" apiKeys " : args . api_keys ,
2021-10-30 17:31:50 +02:00
" keyRequired " : bool ( args . api_keys and args . require_api_key_origin ) ,
2021-10-09 15:45:58 +02:00
" suggestions " : args . suggestions ,
2021-10-25 17:09:23 +02:00
" filesTranslation " : not args . disable_files_translation ,
" supportedFilesFormat " : [ ] if args . disable_files_translation else frontend_argos_supported_files_format ,
2021-05-18 05:41:02 +02:00
" language " : {
" source " : {
" code " : frontend_argos_language_source . code ,
2023-01-05 20:12:35 +02:00
" name " : _lazy ( frontend_argos_language_source . name ) ,
2021-05-18 05:41:02 +02:00
} ,
" target " : {
2023-01-15 19:07:51 +02:00
" code " : target_lang . code ,
" name " : _lazy ( target_lang . name ) ,
2021-05-18 05:41:02 +02:00
} ,
} ,
}
)
2020-12-20 22:05:22 +02:00
2022-12-31 06:23:50 +02:00
@bp.post ( " /suggest " )
2021-10-09 11:25:56 +02:00
def suggest ( ) :
2021-10-09 16:04:16 +02:00
"""
Submit a suggestion to improve a translation
- - -
tags :
- feedback
parameters :
- in : formData
name : q
schema :
type : string
example : Hello world !
required : true
description : Original text
- in : formData
name : s
schema :
type : string
example : ¡ Hola mundo !
required : true
description : Suggested translation
- in : formData
name : source
schema :
type : string
example : en
required : true
description : Language of original text
- in : formData
name : target
schema :
type : string
example : es
required : true
description : Language of suggested translation
responses :
200 :
description : Success
schema :
id : suggest - response
type : object
properties :
success :
type : boolean
description : Whether submission was successful
403 :
description : Not authorized
schema :
id : error - response
type : object
properties :
error :
type : string
description : Error message
"""
if not args . suggestions :
2023-01-05 20:12:35 +02:00
abort ( 403 , description = _ ( " Suggestions are disabled on this server. " ) )
2021-10-09 15:45:58 +02:00
2023-09-29 22:49:52 +02:00
if request . is_json :
json = get_json_dict ( request )
q = json . get ( " q " )
s = json . get ( " s " )
source_lang = json . get ( " source " )
target_lang = json . get ( " target " )
else :
q = request . values . get ( " q " )
s = request . values . get ( " s " )
source_lang = request . values . get ( " source " )
target_lang = request . values . get ( " target " )
2021-10-09 11:44:00 +02:00
2022-02-20 19:22:12 +02:00
if not q :
2023-01-05 20:12:35 +02:00
abort ( 400 , description = _ ( " Invalid request: missing %(name)s parameter " , name = ' q ' ) )
2022-02-20 19:22:12 +02:00
if not s :
2023-01-05 20:12:35 +02:00
abort ( 400 , description = _ ( " Invalid request: missing %(name)s parameter " , name = ' s ' ) )
2022-02-20 19:22:12 +02:00
if not source_lang :
2023-01-05 20:12:35 +02:00
abort ( 400 , description = _ ( " Invalid request: missing %(name)s parameter " , name = ' source ' ) )
2022-02-20 19:22:12 +02:00
if not target_lang :
2023-01-05 20:12:35 +02:00
abort ( 400 , description = _ ( " Invalid request: missing %(name)s parameter " , name = ' target ' ) )
2022-02-20 19:22:12 +02:00
2021-10-09 11:44:00 +02:00
SuggestionsDatabase ( ) . add ( q , s , source_lang , target_lang )
2021-10-09 11:25:56 +02:00
return jsonify ( { " success " : True } )
2022-12-31 06:23:50 +02:00
app = Flask ( __name__ )
2023-01-06 18:50:51 +02:00
app . config [ " SESSION_TYPE " ] = " filesystem "
app . config [ " SESSION_FILE_DIR " ] = os . path . join ( " db " , " sessions " )
2023-05-29 22:47:32 +02:00
app . config [ " JSON_AS_ASCII " ] = False
2023-01-06 18:50:51 +02:00
Session ( app )
2022-12-31 06:23:50 +02:00
if args . debug :
app . config [ " TEMPLATES_AUTO_RELOAD " ] = True
if args . url_prefix :
app . register_blueprint ( bp , url_prefix = args . url_prefix )
else :
app . register_blueprint ( bp )
2023-07-08 04:34:53 +02:00
2023-01-01 20:18:00 +02:00
limiter . init_app ( app )
2022-12-31 06:23:50 +02:00
2021-10-09 16:04:16 +02:00
swag = swagger ( app )
2022-12-31 23:44:25 +02:00
swag [ " info " ] [ " version " ] = get_version ( )
2021-10-09 16:04:16 +02:00
swag [ " info " ] [ " title " ] = " LibreTranslate "
2023-07-09 12:38:03 +02:00
@app.route ( api_url )
2021-10-09 16:04:16 +02:00
@limiter.exempt
def spec ( ) :
2023-01-05 21:07:39 +02:00
return jsonify ( lazy_swag ( swag ) )
2021-10-09 16:04:16 +02:00
2023-01-05 00:54:07 +02:00
app . config [ " BABEL_TRANSLATION_DIRECTORIES " ] = ' locales '
2023-04-27 18:21:04 +02:00
2023-01-04 19:15:18 +02:00
def get_locale ( ) :
2023-01-06 18:50:51 +02:00
override_lang = request . headers . get ( ' X-Override-Accept-Language ' )
if override_lang and override_lang in get_available_locale_codes ( ) :
return override_lang
return session . get ( ' preferred_lang ' , request . accept_languages . best_match ( get_available_locale_codes ( ) ) )
2023-01-04 19:15:18 +02:00
2023-07-09 12:29:11 +02:00
Babel ( app , locale_selector = get_locale )
2023-04-27 18:21:04 +02:00
2023-01-05 00:54:07 +02:00
app . jinja_env . globals . update ( _e = gettext_escaped , _h = gettext_html )
2023-01-04 19:15:18 +02:00
2020-12-20 22:05:22 +02:00
# Call factory function to create our blueprint
2023-07-09 12:38:03 +02:00
swaggerui_blueprint = get_swaggerui_blueprint ( swagger_url , api_url )
2022-12-31 06:23:50 +02:00
if args . url_prefix :
2023-07-09 12:38:03 +02:00
app . register_blueprint ( swaggerui_blueprint , url_prefix = swagger_url )
2022-12-31 06:23:50 +02:00
else :
app . register_blueprint ( swaggerui_blueprint )
2020-12-20 22:05:22 +02:00
2020-12-21 01:17:06 +02:00
return app