mirror of
https://github.com/LibreTranslate/LibreTranslate.git
synced 2024-12-18 08:27:03 +02:00
Merge branch 'main' of https://github.com/ecxod/LibreTranslate into christian
This commit is contained in:
commit
dd4b2cb0b9
@ -11,6 +11,8 @@ RUN apt-get update -qq \
|
|||||||
&& apt-get clean \
|
&& apt-get clean \
|
||||||
&& rm -rf /var/lib/apt
|
&& rm -rf /var/lib/apt
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get upgrade --assume-yes
|
||||||
|
|
||||||
RUN pip install --upgrade pip
|
RUN pip install --upgrade pip
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
25
README.md
25
README.md
@ -164,6 +164,18 @@ docker-compose up -d --build
|
|||||||
|
|
||||||
> Feel free to change the [`docker-compose.yml`](https://github.com/LibreTranslate/LibreTranslate/blob/main/docker-compose.yml) file to adapt it to your deployment needs, or use an extra `docker-compose.prod.yml` file for your deployment configuration.
|
> Feel free to change the [`docker-compose.yml`](https://github.com/LibreTranslate/LibreTranslate/blob/main/docker-compose.yml) file to adapt it to your deployment needs, or use an extra `docker-compose.prod.yml` file for your deployment configuration.
|
||||||
|
|
||||||
|
> The models are stored inside the container under `/root/.local/share` and `/root/.local/cache`. Feel free to use volumes if you do not want to redownload the models when the container is destroyed. Be aware that this will prevent the models from being updated!
|
||||||
|
|
||||||
|
### CUDA
|
||||||
|
|
||||||
|
You can use hardware acceleration to speed up translations on a GPU machine with CUDA 11.2 and [nvidia-docker](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html) installed.
|
||||||
|
|
||||||
|
Run this version with:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker-compose -f docker-compose.cuda.yml up -d --build
|
||||||
|
```
|
||||||
|
|
||||||
## Arguments
|
## Arguments
|
||||||
|
|
||||||
| Argument | Description | Default | Env. name |
|
| Argument | Description | Default | Env. name |
|
||||||
@ -180,8 +192,12 @@ docker-compose up -d --build
|
|||||||
| --frontend-language-target | Set frontend default language - target | `es` | LT_FRONTEND_LANGUAGE_TARGET |
|
| --frontend-language-target | Set frontend default language - target | `es` | LT_FRONTEND_LANGUAGE_TARGET |
|
||||||
| --frontend-timeout | Set frontend translation timeout | `500` | LT_FRONTEND_TIMEOUT |
|
| --frontend-timeout | Set frontend translation timeout | `500` | LT_FRONTEND_TIMEOUT |
|
||||||
| --api-keys | Enable API keys database for per-user rate limits lookup | `Don't use API keys` | LT_API_KEYS |
|
| --api-keys | Enable API keys database for per-user rate limits lookup | `Don't use API keys` | LT_API_KEYS |
|
||||||
|
| --api-keys-db-path | Use a specific path inside the container for the local database. Can be absolute or relative | `api_keys.db` | LT_API_KEYS_DB_PATH |
|
||||||
|
| --api-keys-remote | Use this remote endpoint to query for valid API keys instead of using the local database | `Use local API key database` | LT_API_KEYS_REMOTE |
|
||||||
|
| --get-api-key-link | Show a link in the UI where to direct users to get an API key | `Don't show a link` | LT_GET_API_KEY_LINK |
|
||||||
| --require-api-key-origin | Require use of an API key for programmatic access to the API, unless the request origin matches this domain | `No restrictions on domain origin` | LT_REQUIRE_API_KEY_ORIGIN |
|
| --require-api-key-origin | Require use of an API key for programmatic access to the API, unless the request origin matches this domain | `No restrictions on domain origin` | LT_REQUIRE_API_KEY_ORIGIN |
|
||||||
| --load-only | Set available languages | `all from argostranslate` | LT_LOAD_ONLY |
|
| --load-only | Set available languages | `all from argostranslate` | LT_LOAD_ONLY |
|
||||||
|
| --threads | Set number of threads | `4` | LT_THREADS |
|
||||||
| --suggestions | Allow user suggestions | `false` | LT_SUGGESTIONS |
|
| --suggestions | Allow user suggestions | `false` | LT_SUGGESTIONS |
|
||||||
| --disable-files-translation | Disable files translation | `false` | LT_DISABLE_FILES_TRANSLATION |
|
| --disable-files-translation | Disable files translation | `false` | LT_DISABLE_FILES_TRANSLATION |
|
||||||
| --disable-web-ui | Disable web ui | `false` | LT_DISABLE_WEB_UI |
|
| --disable-web-ui | Disable web ui | `false` | LT_DISABLE_WEB_UI |
|
||||||
@ -210,7 +226,7 @@ See ["LibreTranslate: your own translation service on Kubernetes" by JM Robles](
|
|||||||
|
|
||||||
LibreTranslate supports per-user limit quotas, e.g. you can issue API keys to users so that they can enjoy higher requests limits per minute (if you also set `--req-limit`). By default all users are rate-limited based on `--req-limit`, but passing an optional `api_key` parameter to the REST endpoints allows a user to enjoy higher request limits.
|
LibreTranslate supports per-user limit quotas, e.g. you can issue API keys to users so that they can enjoy higher requests limits per minute (if you also set `--req-limit`). By default all users are rate-limited based on `--req-limit`, but passing an optional `api_key` parameter to the REST endpoints allows a user to enjoy higher request limits.
|
||||||
|
|
||||||
To use API keys simply start LibreTranslate with the `--api-keys` option.
|
To use API keys simply start LibreTranslate with the `--api-keys` option. If you modified the API keys database path with the option `--api-keys-db-path`, you must specify the path with the same argument flag when using the `ltmanage keys` command.
|
||||||
|
|
||||||
### Add New Keys
|
### Add New Keys
|
||||||
|
|
||||||
@ -220,6 +236,11 @@ To issue a new API key with 120 requests per minute limits:
|
|||||||
ltmanage keys add 120
|
ltmanage keys add 120
|
||||||
```
|
```
|
||||||
|
|
||||||
|
If you changed the API keys database path:
|
||||||
|
```bash
|
||||||
|
ltmanage keys --api-keys-db-path path/to/db/dbName.db add 120
|
||||||
|
```
|
||||||
|
|
||||||
### Remove Keys
|
### Remove Keys
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@ -282,9 +303,9 @@ URL |API Key Required|Payment Link|Cost
|
|||||||
[libretranslate.de](https://libretranslate.de)|-|-
|
[libretranslate.de](https://libretranslate.de)|-|-
|
||||||
[translate.argosopentech.com](https://translate.argosopentech.com/)|-|-
|
[translate.argosopentech.com](https://translate.argosopentech.com/)|-|-
|
||||||
[translate.api.skitzen.com](https://translate.api.skitzen.com/)|-|-
|
[translate.api.skitzen.com](https://translate.api.skitzen.com/)|-|-
|
||||||
[libretranslate.pussthecat.org](https://libretranslate.pussthecat.org/)|-|-
|
|
||||||
[translate.fortytwo-it.com](https://translate.fortytwo-it.com/)|-|-
|
[translate.fortytwo-it.com](https://translate.fortytwo-it.com/)|-|-
|
||||||
[translate.terraprint.co](https://translate.terraprint.co/)|-|-
|
[translate.terraprint.co](https://translate.terraprint.co/)|-|-
|
||||||
|
[lt.vern.cc](https://lt.vern.cc)|-|-
|
||||||
|
|
||||||
## Adding New Languages
|
## Adding New Languages
|
||||||
|
|
||||||
|
@ -1,13 +1,18 @@
|
|||||||
|
import os
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import uuid
|
import uuid
|
||||||
|
import requests
|
||||||
from expiringdict import ExpiringDict
|
from expiringdict import ExpiringDict
|
||||||
|
from app.default_values import DEFAULT_ARGUMENTS as DEFARGS
|
||||||
|
|
||||||
DEFAULT_DB_PATH = "api_keys.db"
|
DEFAULT_DB_PATH = DEFARGS['API_KEYS_DB_PATH']
|
||||||
|
|
||||||
|
|
||||||
class Database:
|
class Database:
|
||||||
def __init__(self, db_path=DEFAULT_DB_PATH, max_cache_len=1000, max_cache_age=30):
|
def __init__(self, db_path=DEFAULT_DB_PATH, max_cache_len=1000, max_cache_age=30):
|
||||||
|
db_dir = os.path.dirname(db_path)
|
||||||
|
if not db_dir == "" and not os.path.exists(db_dir):
|
||||||
|
os.makedirs(db_dir)
|
||||||
self.db_path = db_path
|
self.db_path = db_path
|
||||||
self.cache = ExpiringDict(max_len=max_cache_len, max_age_seconds=max_cache_age)
|
self.cache = ExpiringDict(max_len=max_cache_len, max_age_seconds=max_cache_age)
|
||||||
|
|
||||||
@ -61,3 +66,27 @@ class Database:
|
|||||||
def all(self):
|
def all(self):
|
||||||
row = self.c.execute("SELECT api_key, req_limit FROM api_keys")
|
row = self.c.execute("SELECT api_key, req_limit FROM api_keys")
|
||||||
return row.fetchall()
|
return row.fetchall()
|
||||||
|
|
||||||
|
|
||||||
|
class RemoteDatabase:
|
||||||
|
def __init__(self, url, max_cache_len=1000, max_cache_age=600):
|
||||||
|
self.url = url
|
||||||
|
self.cache = ExpiringDict(max_len=max_cache_len, max_age_seconds=max_cache_age)
|
||||||
|
|
||||||
|
def lookup(self, api_key):
|
||||||
|
req_limit = self.cache.get(api_key)
|
||||||
|
if req_limit is None:
|
||||||
|
try:
|
||||||
|
r = requests.post(self.url, data={'api_key': api_key})
|
||||||
|
res = r.json()
|
||||||
|
except Exception as e:
|
||||||
|
print("Cannot authenticate API key: " + str(e))
|
||||||
|
return None
|
||||||
|
|
||||||
|
if res.get('error', None) is None:
|
||||||
|
req_limit = res.get('req_limit', None)
|
||||||
|
else:
|
||||||
|
req_limit = None
|
||||||
|
self.cache[api_key] = req_limit
|
||||||
|
|
||||||
|
return req_limit
|
||||||
|
30
app/app.py
30
app/app.py
@ -17,7 +17,7 @@ from werkzeug.utils import secure_filename
|
|||||||
from app import flood, remove_translated_files, security
|
from app import flood, remove_translated_files, security
|
||||||
from app.language import detect_languages, transliterate
|
from app.language import detect_languages, transliterate
|
||||||
|
|
||||||
from .api_keys import Database
|
from .api_keys import Database, RemoteDatabase
|
||||||
from .suggestions import Database as SuggestionsDatabase
|
from .suggestions import Database as SuggestionsDatabase
|
||||||
|
|
||||||
|
|
||||||
@ -146,7 +146,12 @@ def create_app(args):
|
|||||||
api_keys_db = None
|
api_keys_db = None
|
||||||
|
|
||||||
if args.req_limit > 0 or args.api_keys or args.daily_req_limit > 0:
|
if args.req_limit > 0 or args.api_keys or args.daily_req_limit > 0:
|
||||||
api_keys_db = Database() if args.api_keys else None
|
api_keys_db = None
|
||||||
|
if args.api_keys:
|
||||||
|
if args.api_keys_remote:
|
||||||
|
api_keys_db = RemoteDatabase(args.api_keys_remote)
|
||||||
|
else:
|
||||||
|
api_keys_db = Database(args.api_keys_db_path)
|
||||||
|
|
||||||
from flask_limiter import Limiter
|
from flask_limiter import Limiter
|
||||||
|
|
||||||
@ -190,9 +195,12 @@ def create_app(args):
|
|||||||
and api_keys_db.lookup(ak) is None
|
and api_keys_db.lookup(ak) is None
|
||||||
and request.headers.get("Origin") != args.require_api_key_origin
|
and request.headers.get("Origin") != args.require_api_key_origin
|
||||||
):
|
):
|
||||||
|
description = "Please contact the server operator to get an API key"
|
||||||
|
if args.get_api_key_link:
|
||||||
|
description = "Visit %s to get an API key" % args.get_api_key_link
|
||||||
abort(
|
abort(
|
||||||
403,
|
403,
|
||||||
description="Please contact the server operator to obtain an API key",
|
description=description,
|
||||||
)
|
)
|
||||||
|
|
||||||
return f(*a, **kw)
|
return f(*a, **kw)
|
||||||
@ -227,6 +235,7 @@ def create_app(args):
|
|||||||
gaId=args.ga_id,
|
gaId=args.ga_id,
|
||||||
frontendTimeout=args.frontend_timeout,
|
frontendTimeout=args.frontend_timeout,
|
||||||
api_keys=args.api_keys,
|
api_keys=args.api_keys,
|
||||||
|
get_api_key_link=args.get_api_key_link,
|
||||||
web_version=os.environ.get("LT_WEB") is not None,
|
web_version=os.environ.get("LT_WEB") is not None,
|
||||||
version=get_version()
|
version=get_version()
|
||||||
)
|
)
|
||||||
@ -475,6 +484,21 @@ def create_app(args):
|
|||||||
abort(400, description="%s format is not supported" % text_format)
|
abort(400, description="%s format is not supported" % text_format)
|
||||||
|
|
||||||
def improve_translation(source, translation):
|
def improve_translation(source, translation):
|
||||||
|
source = source.strip()
|
||||||
|
|
||||||
|
source_last_char = source[len(source) - 1]
|
||||||
|
translation_last_char = translation[len(translation) - 1]
|
||||||
|
|
||||||
|
punctuation_chars = ['!', '?', '.', ',', ';']
|
||||||
|
if source_last_char in punctuation_chars:
|
||||||
|
if translation_last_char != source_last_char:
|
||||||
|
if translation_last_char in punctuation_chars:
|
||||||
|
translation = translation[:-1]
|
||||||
|
|
||||||
|
translation += source_last_char
|
||||||
|
elif translation_last_char in punctuation_chars:
|
||||||
|
translation = translation[:-1]
|
||||||
|
|
||||||
if source.islower():
|
if source.islower():
|
||||||
return translation.lower()
|
return translation.lower()
|
||||||
|
|
||||||
|
@ -106,6 +106,21 @@ _default_options_objects = [
|
|||||||
'default_value': False,
|
'default_value': False,
|
||||||
'value_type': 'bool'
|
'value_type': 'bool'
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
'name': 'API_KEYS_DB_PATH',
|
||||||
|
'default_value': 'api_keys.db',
|
||||||
|
'value_type': 'str'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'name': 'API_KEYS_REMOTE',
|
||||||
|
'default_value': '',
|
||||||
|
'value_type': 'str'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'name': 'GET_API_KEY_LINK',
|
||||||
|
'default_value': '',
|
||||||
|
'value_type': 'str'
|
||||||
|
},
|
||||||
{
|
{
|
||||||
'name': 'REQUIRE_API_KEY_ORIGIN',
|
'name': 'REQUIRE_API_KEY_ORIGIN',
|
||||||
'default_value': '',
|
'default_value': '',
|
||||||
@ -116,6 +131,11 @@ _default_options_objects = [
|
|||||||
'default_value': None,
|
'default_value': None,
|
||||||
'value_type': 'str'
|
'value_type': 'str'
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
'name': 'THREADS',
|
||||||
|
'default_value': 4,
|
||||||
|
'value_type': 'int'
|
||||||
|
},
|
||||||
{
|
{
|
||||||
'name': 'SUGGESTIONS',
|
'name': 'SUGGESTIONS',
|
||||||
'default_value': False,
|
'default_value': False,
|
||||||
|
@ -31,7 +31,7 @@ def setup(violations_threshold=100):
|
|||||||
threshold = violations_threshold
|
threshold = violations_threshold
|
||||||
|
|
||||||
scheduler = BackgroundScheduler()
|
scheduler = BackgroundScheduler()
|
||||||
scheduler.add_job(func=forgive_banned, trigger="interval", minutes=480)
|
scheduler.add_job(func=forgive_banned, trigger="interval", minutes=30)
|
||||||
scheduler.start()
|
scheduler.start()
|
||||||
|
|
||||||
# Shut down the scheduler when exiting the app
|
# Shut down the scheduler when exiting the app
|
||||||
|
@ -65,7 +65,7 @@ def check_and_install_models(force=False, load_only_lang_codes=None):
|
|||||||
def check_and_install_transliteration(force=False):
|
def check_and_install_transliteration(force=False):
|
||||||
# 'en' is not a supported transliteration language
|
# 'en' is not a supported transliteration language
|
||||||
transliteration_languages = [
|
transliteration_languages = [
|
||||||
l.code for l in app.language.languages if l.code != "en"
|
l.code for l in app.language.load_languages() if l.code != "en"
|
||||||
]
|
]
|
||||||
|
|
||||||
# check installed
|
# check installed
|
||||||
|
31
app/main.py
31
app/main.py
@ -89,6 +89,24 @@ def get_args():
|
|||||||
action="store_true",
|
action="store_true",
|
||||||
help="Enable API keys database for per-user rate limits lookup",
|
help="Enable API keys database for per-user rate limits lookup",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--api-keys-db-path",
|
||||||
|
default=DEFARGS['API_KEYS_DB_PATH'],
|
||||||
|
type=str,
|
||||||
|
help="Use a specific path inside the container for the local database. Can be absolute or relative (%(default)s)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--api-keys-remote",
|
||||||
|
default=DEFARGS['API_KEYS_REMOTE'],
|
||||||
|
type=str,
|
||||||
|
help="Use this remote endpoint to query for valid API keys instead of using the local database",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--get-api-key-link",
|
||||||
|
default=DEFARGS['GET_API_KEY_LINK'],
|
||||||
|
type=str,
|
||||||
|
help="Show a link in the UI where to direct users to get an API key",
|
||||||
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--require-api-key-origin",
|
"--require-api-key-origin",
|
||||||
type=str,
|
type=str,
|
||||||
@ -102,6 +120,13 @@ def get_args():
|
|||||||
metavar="<comma-separated language codes>",
|
metavar="<comma-separated language codes>",
|
||||||
help="Set available languages (ar,de,en,es,fr,ga,hi,it,ja,ko,pt,ru,zh)",
|
help="Set available languages (ar,de,en,es,fr,ga,hi,it,ja,ko,pt,ru,zh)",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--threads",
|
||||||
|
default=DEFARGS['THREADS'],
|
||||||
|
type=int,
|
||||||
|
metavar="<number of threads>",
|
||||||
|
help="Set number of threads (%(default)s)",
|
||||||
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--suggestions", default=DEFARGS['SUGGESTIONS'], action="store_true", help="Allow user suggestions"
|
"--suggestions", default=DEFARGS['SUGGESTIONS'], action="store_true", help="Allow user suggestions"
|
||||||
)
|
)
|
||||||
@ -128,11 +153,15 @@ def main():
|
|||||||
else:
|
else:
|
||||||
from waitress import serve
|
from waitress import serve
|
||||||
|
|
||||||
|
url_scheme = "https" if args.ssl else "http"
|
||||||
|
print("Running on %s://%s:%s" % (url_scheme, args.host, args.port))
|
||||||
|
|
||||||
serve(
|
serve(
|
||||||
app,
|
app,
|
||||||
host=args.host,
|
host=args.host,
|
||||||
port=args.port,
|
port=args.port,
|
||||||
url_scheme="https" if args.ssl else "http",
|
url_scheme=url_scheme,
|
||||||
|
threads=args.threads
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -1,6 +1,8 @@
|
|||||||
import argparse
|
import argparse
|
||||||
|
import os
|
||||||
|
|
||||||
from app.api_keys import Database
|
from app.api_keys import Database
|
||||||
|
from app.default_values import DEFAULT_ARGUMENTS as DEFARGS
|
||||||
|
|
||||||
|
|
||||||
def manage():
|
def manage():
|
||||||
@ -10,6 +12,12 @@ def manage():
|
|||||||
)
|
)
|
||||||
|
|
||||||
keys_parser = subparsers.add_parser("keys", help="Manage API keys database")
|
keys_parser = subparsers.add_parser("keys", help="Manage API keys database")
|
||||||
|
keys_parser.add_argument(
|
||||||
|
"--api-keys-db-path",
|
||||||
|
default=DEFARGS['API_KEYS_DB_PATH'],
|
||||||
|
type=str,
|
||||||
|
help="Use a specific path inside the container for the local database",
|
||||||
|
)
|
||||||
keys_subparser = keys_parser.add_subparsers(
|
keys_subparser = keys_parser.add_subparsers(
|
||||||
help="", dest="sub_command", title="Command List"
|
help="", dest="sub_command", title="Command List"
|
||||||
)
|
)
|
||||||
@ -30,7 +38,10 @@ def manage():
|
|||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
if args.command == "keys":
|
if args.command == "keys":
|
||||||
db = Database()
|
if not os.path.exists(args.api_keys_db_path):
|
||||||
|
print("No such database: %s" % args.api_keys_db_path)
|
||||||
|
exit(1)
|
||||||
|
db = Database(args.api_keys_db_path)
|
||||||
if args.sub_command is None:
|
if args.sub_command is None:
|
||||||
# Print keys
|
# Print keys
|
||||||
keys = db.all()
|
keys = db.all()
|
||||||
|
@ -8,7 +8,7 @@
|
|||||||
background-color: #1E5DA6 !important;
|
background-color: #1E5DA6 !important;
|
||||||
}
|
}
|
||||||
|
|
||||||
/*" like in btn-delete-text */
|
/* like in btn-delete-text */
|
||||||
.btn-flat {
|
.btn-flat {
|
||||||
color: #666;
|
color: #666;
|
||||||
}
|
}
|
||||||
@ -30,6 +30,7 @@
|
|||||||
color: #fff;
|
color: #fff;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* like in textarea */
|
||||||
.card-content {
|
.card-content {
|
||||||
border: 1px solid #444 !important;
|
border: 1px solid #444 !important;
|
||||||
background-color: #222 !important;
|
background-color: #222 !important;
|
||||||
@ -57,6 +58,7 @@
|
|||||||
background-color: #222 !important;
|
background-color: #222 !important;
|
||||||
color: #fff;
|
color: #fff;
|
||||||
}
|
}
|
||||||
|
/* like in file dropzone */
|
||||||
.textarea-container {
|
.textarea-container {
|
||||||
margin-top: 1rem;
|
margin-top: 1rem;
|
||||||
}
|
}
|
||||||
|
@ -40,13 +40,16 @@ document.addEventListener('DOMContentLoaded', function(){
|
|||||||
frontendTimeout: 500
|
frontendTimeout: 500
|
||||||
},
|
},
|
||||||
mounted: function() {
|
mounted: function() {
|
||||||
var self = this;
|
const self = this;
|
||||||
var requestSettings = new XMLHttpRequest();
|
|
||||||
requestSettings.open('GET', BaseUrl + '/frontend/settings', true);
|
|
||||||
|
|
||||||
requestSettings.onload = function() {
|
const settingsRequest = new XMLHttpRequest();
|
||||||
|
settingsRequest.open("GET", BaseUrl + "/frontend/settings", true);
|
||||||
|
|
||||||
|
const langsRequest = new XMLHttpRequest();
|
||||||
|
langsRequest.open("GET", BaseUrl + "/languages", true);
|
||||||
|
|
||||||
|
settingsRequest.onload = function() {
|
||||||
if (this.status >= 200 && this.status < 400) {
|
if (this.status >= 200 && this.status < 400) {
|
||||||
// Success!
|
|
||||||
self.settings = JSON.parse(this.response);
|
self.settings = JSON.parse(this.response);
|
||||||
self.sourceLang = self.settings.language.source.code;
|
self.sourceLang = self.settings.language.source.code;
|
||||||
self.targetLang = self.settings.language.target.code;
|
self.targetLang = self.settings.language.target.code;
|
||||||
@ -55,64 +58,32 @@ document.addEventListener('DOMContentLoaded', function(){
|
|||||||
self.supportedFilesFormat = self.settings.supportedFilesFormat;
|
self.supportedFilesFormat = self.settings.supportedFilesFormat;
|
||||||
self.filesTranslation = self.settings.filesTranslation;
|
self.filesTranslation = self.settings.filesTranslation;
|
||||||
self.frontendTimeout = self.settings.frontendTimeout;
|
self.frontendTimeout = self.settings.frontendTimeout;
|
||||||
|
|
||||||
|
if (langsRequest.response) {
|
||||||
|
handleLangsResponse(self, langsRequest);
|
||||||
|
} else {
|
||||||
|
langsRequest.onload = function() {
|
||||||
|
handleLangsResponse(self, this);
|
||||||
|
}
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
self.error = "Cannot load /frontend/settings";
|
self.error = "Cannot load /frontend/settings";
|
||||||
self.loading = false;
|
self.loading = false;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
requestSettings.onerror = function() {
|
settingsRequest.onerror = function() {
|
||||||
self.error = "Error while calling /frontend/settings";
|
self.error = "Error while calling /frontend/settings";
|
||||||
self.loading = false;
|
self.loading = false;
|
||||||
};
|
};
|
||||||
|
|
||||||
requestSettings.send();
|
langsRequest.onerror = function() {
|
||||||
|
|
||||||
var requestLanguages = new XMLHttpRequest();
|
|
||||||
requestLanguages.open('GET', BaseUrl + '/languages', true);
|
|
||||||
|
|
||||||
requestLanguages.onload = function() {
|
|
||||||
if (this.status >= 200 && this.status < 400) {
|
|
||||||
// Success!
|
|
||||||
self.langs = JSON.parse(this.response);
|
|
||||||
self.langs.push({ name: 'Auto Detect (Experimental)', code: 'auto' })
|
|
||||||
if (self.langs.length === 0){
|
|
||||||
self.loading = false;
|
|
||||||
self.error = "No languages available. Did you install the models correctly?"
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const sourceLanguage = self.langs.find(l => l.code === self.getQueryParam('source'))
|
|
||||||
const isSourceAuto = !sourceLanguage && self.getQueryParam('source') === "auto"
|
|
||||||
const targetLanguage = self.langs.find(l => l.code === self.getQueryParam('target'))
|
|
||||||
|
|
||||||
if (sourceLanguage || isSourceAuto) {
|
|
||||||
self.sourceLang = isSourceAuto ? "auto" : sourceLanguage.code
|
|
||||||
}
|
|
||||||
|
|
||||||
if (targetLanguage) {
|
|
||||||
self.targetLang = targetLanguage.code
|
|
||||||
}
|
|
||||||
|
|
||||||
const defaultText = self.getQueryParam('q')
|
|
||||||
|
|
||||||
if(defaultText) {
|
|
||||||
self.inputText = decodeURI(defaultText)
|
|
||||||
}
|
|
||||||
|
|
||||||
self.loading = false;
|
|
||||||
} else {
|
|
||||||
self.error = "Cannot load /languages";
|
|
||||||
self.loading = false;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
requestLanguages.onerror = function() {
|
|
||||||
self.error = "Error while calling /languages";
|
self.error = "Error while calling /languages";
|
||||||
self.loading = false;
|
self.loading = false;
|
||||||
};
|
};
|
||||||
|
|
||||||
requestLanguages.send();
|
settingsRequest.send();
|
||||||
|
langsRequest.send();
|
||||||
},
|
},
|
||||||
updated: function(){
|
updated: function(){
|
||||||
M.FormSelect.init(this.$refs.sourceLangDropdown);
|
M.FormSelect.init(this.$refs.sourceLangDropdown);
|
||||||
@ -137,27 +108,11 @@ document.addEventListener('DOMContentLoaded', function(){
|
|||||||
// but properly display checkmarks on supported browsers.
|
// but properly display checkmarks on supported browsers.
|
||||||
// Also change the <select> width value depending on the <option> length
|
// Also change the <select> width value depending on the <option> length
|
||||||
if (this.$refs.sourceLangDropdown) {
|
if (this.$refs.sourceLangDropdown) {
|
||||||
for (var i = 0; i < this.$refs.sourceLangDropdown.children.length; i++){
|
updateSelectedAttribute(this.$refs.sourceLangDropdown, this.sourceLang);
|
||||||
var el = this.$refs.sourceLangDropdown.children[i];
|
|
||||||
if (el.value === this.sourceLang){
|
|
||||||
el.setAttribute('selected', '');
|
|
||||||
this.$refs.sourceLangDropdown.style.width = getTextWidth(el.text) + 24 + 'px';
|
|
||||||
}else{
|
|
||||||
el.removeAttribute('selected');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this.$refs.targetLangDropdown) {
|
if (this.$refs.targetLangDropdown) {
|
||||||
for (var i = 0; i < this.$refs.targetLangDropdown.children.length; i++){
|
updateSelectedAttribute(this.$refs.targetLangDropdown, this.targetLang);
|
||||||
var el = this.$refs.targetLangDropdown.children[i];
|
|
||||||
if (el.value === this.targetLang){
|
|
||||||
el.setAttribute('selected', '');
|
|
||||||
this.$refs.targetLangDropdown.style.width = getTextWidth(el.text) + 24 + 'px';
|
|
||||||
}else{
|
|
||||||
el.removeAttribute('selected');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
computed: {
|
computed: {
|
||||||
@ -168,7 +123,8 @@ document.addEventListener('DOMContentLoaded', function(){
|
|||||||
' q: ' + this.$options.filters.escape(this.inputText) + ',',
|
' q: ' + this.$options.filters.escape(this.inputText) + ',',
|
||||||
' source: ' + this.$options.filters.escape(this.sourceLang) + ',',
|
' source: ' + this.$options.filters.escape(this.sourceLang) + ',',
|
||||||
' target: ' + this.$options.filters.escape(this.targetLang) + ',',
|
' target: ' + this.$options.filters.escape(this.targetLang) + ',',
|
||||||
' format: "' + (this.isHtml ? "html" : "text") + '"',
|
' format: "' + (this.isHtml ? "html" : "text") + '",',
|
||||||
|
' api_key: "' + (localStorage.getItem("api_key") || "") + '"',
|
||||||
' }),',
|
' }),',
|
||||||
' headers: { "Content-Type": "application/json" }',
|
' headers: { "Content-Type": "application/json" }',
|
||||||
'});',
|
'});',
|
||||||
@ -422,9 +378,62 @@ document.addEventListener('DOMContentLoaded', function(){
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
});
|
});
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param {object} self
|
||||||
|
* @param {XMLHttpRequest} response
|
||||||
|
*/
|
||||||
|
function handleLangsResponse(self, response) {
|
||||||
|
if (response.status >= 200 && response.status < 400) {
|
||||||
|
self.langs = JSON.parse(response.response);
|
||||||
|
|
||||||
|
if (self.langs.length === 0){
|
||||||
|
self.loading = false;
|
||||||
|
self.error = "No languages available. Did you install the models correctly?"
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
self.langs.push({ name: "Auto Detect (Experimental)", code: "auto" })
|
||||||
|
|
||||||
|
const sourceLanguage = self.langs.find(l => l.code === self.getQueryParam("source"))
|
||||||
|
const targetLanguage = self.langs.find(l => l.code === self.getQueryParam("target"))
|
||||||
|
|
||||||
|
if (sourceLanguage) {
|
||||||
|
self.sourceLang = sourceLanguage.code
|
||||||
|
}
|
||||||
|
|
||||||
|
if (targetLanguage) {
|
||||||
|
self.targetLang = targetLanguage.code
|
||||||
|
}
|
||||||
|
|
||||||
|
const defaultText = self.getQueryParam("q")
|
||||||
|
|
||||||
|
if (defaultText) {
|
||||||
|
self.inputText = decodeURI(defaultText)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
self.error = "Cannot load /languages";
|
||||||
|
}
|
||||||
|
|
||||||
|
self.loading = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param {object} langDropdown
|
||||||
|
* @param {string} lang
|
||||||
|
*/
|
||||||
|
function updateSelectedAttribute(langDropdown, lang) {
|
||||||
|
for (const child of langDropdown.children) {
|
||||||
|
if (child.value === lang){
|
||||||
|
child.setAttribute('selected', '');
|
||||||
|
langDropdown.style.width = getTextWidth(child.text) + 24 + 'px';
|
||||||
|
} else{
|
||||||
|
child.removeAttribute('selected');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function getTextWidth(text) {
|
function getTextWidth(text) {
|
||||||
var canvas = getTextWidth.canvas || (getTextWidth.canvas = document.createElement("canvas"));
|
var canvas = getTextWidth.canvas || (getTextWidth.canvas = document.createElement("canvas"));
|
||||||
var ctx = canvas.getContext("2d");
|
var ctx = canvas.getContext("2d");
|
||||||
@ -436,7 +445,9 @@ function getTextWidth(text) {
|
|||||||
function setApiKey(){
|
function setApiKey(){
|
||||||
var prevKey = localStorage.getItem("api_key") || "";
|
var prevKey = localStorage.getItem("api_key") || "";
|
||||||
var newKey = "";
|
var newKey = "";
|
||||||
newKey = window.prompt("Type in your API Key. If you need an API key, contact the server operator.", prevKey);
|
var instructions = "contact the server operator.";
|
||||||
|
if (window.getApiKeyLink) instructions = "press the \"Get API Key\" link."
|
||||||
|
newKey = window.prompt("Type in your API Key. If you need an API key, " + instructions, prevKey);
|
||||||
if (newKey === null) newKey = "";
|
if (newKey === null) newKey = "";
|
||||||
|
|
||||||
localStorage.setItem("api_key", newKey);
|
localStorage.setItem("api_key", newKey);
|
||||||
|
@ -45,6 +45,10 @@
|
|||||||
</a>
|
</a>
|
||||||
<ul class="right hide-on-med-and-down">
|
<ul class="right hide-on-med-and-down">
|
||||||
<li><a href="/docs">API Docs</a></li>
|
<li><a href="/docs">API Docs</a></li>
|
||||||
|
{% if get_api_key_link %}
|
||||||
|
<li><a href="{{ get_api_key_link }}">Get API Key</a></li>
|
||||||
|
<script>window.getApiKeyLink = "{{ get_api_key_link }}";</script>
|
||||||
|
{% endif %}
|
||||||
<li><a href="https://github.com/LibreTranslate/LibreTranslate" rel="noopener noreferrer">GitHub</a></li>
|
<li><a href="https://github.com/LibreTranslate/LibreTranslate" rel="noopener noreferrer">GitHub</a></li>
|
||||||
{% if api_keys %}
|
{% if api_keys %}
|
||||||
<li><a href="javascript:setApiKey()" title="Set API Key"><i class="material-icons">vpn_key</i></a></li>
|
<li><a href="javascript:setApiKey()" title="Set API Key"><i class="material-icons">vpn_key</i></a></li>
|
||||||
@ -53,6 +57,9 @@
|
|||||||
|
|
||||||
<ul id="nav-mobile" class="sidenav">
|
<ul id="nav-mobile" class="sidenav">
|
||||||
<li><a href="/docs">API Docs</a></li>
|
<li><a href="/docs">API Docs</a></li>
|
||||||
|
{% if get_api_key_link %}
|
||||||
|
<li><a href="{{ get_api_key_link }}">Get API Key</a></li>
|
||||||
|
{% endif %}
|
||||||
<li><a href="https://github.com/LibreTranslate/LibreTranslate" rel="noopener noreferrer">GitHub</a></li>
|
<li><a href="https://github.com/LibreTranslate/LibreTranslate" rel="noopener noreferrer">GitHub</a></li>
|
||||||
{% if api_keys %}
|
{% if api_keys %}
|
||||||
<li><a href="javascript:setApiKey()" title="Set API Key"><i class="material-icons">vpn_key</i></a></li>
|
<li><a href="javascript:setApiKey()" title="Set API Key"><i class="material-icons">vpn_key</i></a></li>
|
||||||
@ -275,7 +282,7 @@
|
|||||||
<p><a class="grey-text text-lighten-4" href="/javascript-licenses" rel="jslicense">JavaScript license information</a></p>
|
<p><a class="grey-text text-lighten-4" href="/javascript-licenses" rel="jslicense">JavaScript license information</a></p>
|
||||||
{% if web_version %}
|
{% if web_version %}
|
||||||
<p>
|
<p>
|
||||||
This public API should be used for testing, personal or infrequent use. If you're going to run an application in production, please <a href="https://github.com/LibreTranslate/LibreTranslate" class="grey-text text-lighten-4" rel="noopener noreferrer">host your own server</a> or <a class="grey-text text-lighten-4" href="https://github.com/LibreTranslate/LibreTranslate#mirrors" rel="noopener noreferrer">get an API key</a>.
|
This public API should be used for testing, personal or infrequent use. If you're going to run an application in production, please <a href="https://github.com/LibreTranslate/LibreTranslate" class="grey-text text-lighten-4" rel="noopener noreferrer">host your own server</a> or <a class="grey-text text-lighten-4" href="{{ get_api_key_link if get_api_key_link else 'https://github.com/LibreTranslate/LibreTranslate#mirrors' }}" rel="noopener noreferrer">get an API key</a>.
|
||||||
</p>
|
</p>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
|
18
docker-compose.cuda.yml
Normal file
18
docker-compose.cuda.yml
Normal file
@ -0,0 +1,18 @@
|
|||||||
|
version: "3"
|
||||||
|
|
||||||
|
services:
|
||||||
|
libretranslate-cuda:
|
||||||
|
container_name: libretranslate-cuda
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
dockerfile: docker/Dockerfile.cuda
|
||||||
|
restart: unless-stopped
|
||||||
|
ports:
|
||||||
|
- 5000:5000
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
reservations:
|
||||||
|
devices:
|
||||||
|
- driver: nvidia
|
||||||
|
count: 1
|
||||||
|
capabilities: [gpu]
|
@ -9,3 +9,11 @@ services:
|
|||||||
- 5000:5000
|
- 5000:5000
|
||||||
## Uncomment above command and define your args if necessary
|
## Uncomment above command and define your args if necessary
|
||||||
# command: --ssl --ga-id MY-GA-ID --req-limit 100 --char-limit 500
|
# command: --ssl --ga-id MY-GA-ID --req-limit 100 --char-limit 500
|
||||||
|
## Uncomment this section and the `volumes` section if you want to backup your API keys
|
||||||
|
# environment:
|
||||||
|
# - LT_API_KEYS_DB_PATH=/app/db/api_keys.db # Same result as `db/api_keys.db` or `./db/api_keys.db`
|
||||||
|
# volumes:
|
||||||
|
# - libretranslate_api_keys:/app/db/api_keys.db
|
||||||
|
|
||||||
|
# volumes:
|
||||||
|
# libretranslate_api_keys:
|
45
docker/Dockerfile.cuda
Normal file
45
docker/Dockerfile.cuda
Normal file
@ -0,0 +1,45 @@
|
|||||||
|
FROM nvidia/cuda:11.2.2-devel-ubuntu20.04
|
||||||
|
|
||||||
|
ENV ARGOS_DEVICE_TYPE cuda
|
||||||
|
ARG with_models=true
|
||||||
|
ARG models=
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
ARG DEBIAN_FRONTEND=noninteractive
|
||||||
|
RUN apt-get update -qq \
|
||||||
|
&& apt-get -qqq install --no-install-recommends -y libicu-dev libaspell-dev libcairo2 libcairo2-dev pkg-config gcc g++ python3.8-dev python3-pip libpython3.8-dev\
|
||||||
|
&& apt-get clean \
|
||||||
|
&& rm -rf /var/lib/apt
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get upgrade --assume-yes
|
||||||
|
|
||||||
|
RUN pip3 install --upgrade pip && apt-get remove python3-pip --assume-yes
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
RUN ln -s /usr/bin/python3 /usr/bin/python
|
||||||
|
|
||||||
|
RUN pip3 install torch==1.12.0+cu116 -f https://download.pytorch.org/whl/torch_stable.html
|
||||||
|
|
||||||
|
RUN if [ "$with_models" = "true" ]; then \
|
||||||
|
# install only the dependencies first
|
||||||
|
pip3 install -e .; \
|
||||||
|
# initialize the language models
|
||||||
|
if [ ! -z "$models" ]; then \
|
||||||
|
./install_models.py --load_only_lang_codes "$models"; \
|
||||||
|
else \
|
||||||
|
./install_models.py; \
|
||||||
|
fi \
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Install package from source code
|
||||||
|
RUN pip3 install . \
|
||||||
|
&& pip3 cache purge
|
||||||
|
|
||||||
|
# Depending on your cuda install you may need to uncomment this line to allow the container to access the cuda libraries
|
||||||
|
# See: https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#post-installation-actions
|
||||||
|
# ENV LD_LIBRARY_PATH=/usr/local/cuda/lib:/usr/local/cuda/lib64
|
||||||
|
|
||||||
|
EXPOSE 5000
|
||||||
|
ENTRYPOINT [ "libretranslate", "--host", "0.0.0.0" ]
|
@ -3,7 +3,7 @@ Flask==2.1.2
|
|||||||
flask-swagger==0.2.14
|
flask-swagger==0.2.14
|
||||||
flask-swagger-ui==3.36.0
|
flask-swagger-ui==3.36.0
|
||||||
Flask-Limiter==2.4.5.1
|
Flask-Limiter==2.4.5.1
|
||||||
waitress==2.1.1
|
waitress==2.1.2
|
||||||
expiringdict==1.2.1
|
expiringdict==1.2.1
|
||||||
pyicu>=2.8
|
pyicu>=2.8
|
||||||
pycld2==0.41
|
pycld2==0.41
|
||||||
@ -12,6 +12,8 @@ polyglot==16.7.4
|
|||||||
appdirs==1.4.4
|
appdirs==1.4.4
|
||||||
APScheduler==3.9.1
|
APScheduler==3.9.1
|
||||||
translatehtml==1.5.2
|
translatehtml==1.5.2
|
||||||
argos-translate-files==1.0.5
|
argos-translate-files==1.1.0
|
||||||
itsdangerous==2.1.2
|
itsdangerous==2.1.2
|
||||||
Werkzeug==2.1.2
|
Werkzeug==2.1.2
|
||||||
|
requests==2.28.0
|
||||||
|
|
||||||
|
46
suggestions-to-jsonl.py
Executable file
46
suggestions-to-jsonl.py
Executable file
@ -0,0 +1,46 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
import argparse
|
||||||
|
import time
|
||||||
|
import sqlite3
|
||||||
|
import json
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
parser = argparse.ArgumentParser(description="Program to generate JSONL files from a LibreTranslate's suggestions.db")
|
||||||
|
parser.add_argument(
|
||||||
|
"--db",
|
||||||
|
type=str,
|
||||||
|
nargs=1,
|
||||||
|
help="Path to suggestions.db file",
|
||||||
|
default='suggestions.db'
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--clear",
|
||||||
|
action='store_true',
|
||||||
|
help="Clear suggestions.db after generation",
|
||||||
|
default=False
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
output_file = str(int(time.time())) + ".jsonl"
|
||||||
|
|
||||||
|
con = sqlite3.connect(args.db, check_same_thread=False)
|
||||||
|
cur = con.cursor()
|
||||||
|
|
||||||
|
with open(output_file, 'w', encoding="utf-8") as f:
|
||||||
|
for row in cur.execute('SELECT q, s, source, target FROM suggestions WHERE source != "auto" ORDER BY source'):
|
||||||
|
q, s, source, target = row
|
||||||
|
obj = {
|
||||||
|
'q': q,
|
||||||
|
's': s,
|
||||||
|
'source': source,
|
||||||
|
'target': target
|
||||||
|
}
|
||||||
|
json.dump(obj, f, ensure_ascii=False)
|
||||||
|
f.write('\n')
|
||||||
|
|
||||||
|
print("Wrote %s" % output_file)
|
||||||
|
|
||||||
|
if args.clear:
|
||||||
|
cur.execute("DELETE FROM suggestions")
|
||||||
|
con.commit()
|
||||||
|
print("Cleared " + args.db)
|
Loading…
Reference in New Issue
Block a user