Merge pull request #462 from vemonet/use-standard-build-system

Update the project configuration to use modern PEP standards, and replace `flake8` by `ruff` for linting

.github/workflows/publish-docker.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1

.github/workflows/publish-package.yml

@@ -9,35 +9,8 @@ on:
 jobs:
 
   tests:
-    runs-on: ubuntu-latest
+    uses: LibreTranslate/LibreTranslate/.github/workflows/run-tests.yml@main
-    strategy:
+    secrets: inherit
-      matrix:
-        python-version: ['3.8', '3.9', '3.10']
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install pytest flake8
-        pip install .
-        python scripts/compile_locales.py
-
-    - name: Check code style with flake8 (lint)
-      run: |
-        # warnings if there are Python syntax errors or undefined names
-        # (remove --exit-zero to fail when syntax error)
-        flake8 . --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-
-    - name: Test with pytest
-      run: pytest
 
 
   publish:
@@ -45,23 +18,23 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: '3.8'
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
+        pip install build
-        pip install setuptools wheel twine
-        python setup.py sdist bdist_wheel
 
-    - name: Build and publish to PyPI
+    - name: Build
-      env:
-        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
       run: |
         pip install Babel==2.11.0
         python scripts/compile_locales.py
-        python setup.py sdist bdist_wheel
+        python -m build
-        twine upload dist/*
+
+    - name: Publish to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        user: ${{ secrets.PYPI_USERNAME }}
+        password: ${{ secrets.PYPI_PASSWORD }}

.github/workflows/run-tests.yml

@@ -2,6 +2,7 @@ name: Run tests
 # Run test at each push to main, if changes to package or tests files
 on:
   workflow_dispatch:
+  workflow_call:
   pull_request:
     branches: [ main ]
   push:
@@ -21,35 +22,26 @@ jobs:
         python-version: ['3.8', '3.9', '3.10']
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
+        pipx install hatch
-        pip install pytest flake8
+        hatch run locales
-        pip install .
-
-    - name: Check code style with flake8 (lint)
-      run: |
-        # warnings if there are Python syntax errors or undefined names
-        # (remove --exit-zero to fail when syntax error)
-        flake8 . --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
 
     - name: Test with pytest
-      run: pytest -v
+      run: hatch run test
 
 
   test_docker_build:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Docker build
         run: docker build -f docker/Dockerfile -t libretranslate .
 

CONTRIBUTING.md

@@ -20,17 +20,57 @@ sudo dnf install cmake
 
 ## Getting Started
 
+Install [`hatch`](https://hatch.pypa.io) to manage the projects dependencies and run dev scripts:
+
+```bash
+pipx install hatch
+```
+
+Clone the repository:
+
 ```bash
 git clone https://github.com/LibreTranslate/LibreTranslate.git
 cd LibreTranslate
-pip install -e .
+```
-libretranslate [args]
 
+Run in development:
+
+```bash
+hatch run dev
+```
+
+Then open a web browser to <http://localhost:5000>
+
+You can also start a new shell in a virtual environment with libretranslate installed:
+
+```bash
+hatch shell
+libretranslate [args]
 # Or
 python main.py [args]
 ```
 
-Then open a web browser to <http://localhost:5000>
+> You can still use `pip install -e ".[test]"` directly if you don't want to use hatch.
+
+## Run the tests
+
+Run the test suite and linting checks:
+
+```bash
+hatch run test
+```
+
+To display all `print()` when debugging:
+
+```bash
+hatch run test -s
+```
+
+You can also run the tests on multiple python versions:
+
+```bash
+hatch run all:test
+```
 
 ## Run with Docker
 

libretranslate/api_keys.py

@@ -1,8 +1,10 @@
 import os
 import sqlite3
 import uuid
+
 import requests
 from expiringdict import ExpiringDict
+
 from libretranslate.default_values import DEFAULT_ARGUMENTS as DEFARGS
 
 DEFAULT_DB_PATH = DEFARGS['API_KEYS_DB_PATH']
@@ -12,14 +14,14 @@ class Database:
     def __init__(self, db_path=DEFAULT_DB_PATH, max_cache_len=1000, max_cache_age=30):
         # Legacy check - this can be removed at some point in the near future
         if os.path.isfile("api_keys.db") and not os.path.isfile("db/api_keys.db"):
-            print("Migrating %s to %s" % ("api_keys.db", "db/api_keys.db"))
+            print("Migrating {} to {}".format("api_keys.db", "db/api_keys.db"))
             try:
                 os.rename("api_keys.db", "db/api_keys.db")
             except Exception as e:
                 print(str(e))
 
         db_dir = os.path.dirname(db_path)
-        if not db_dir == "" and not os.path.exists(db_dir):
+        if db_dir != '' and not os.path.exists(db_dir):
             os.makedirs(db_dir)
         self.db_path = db_path
         self.cache = ExpiringDict(max_len=max_cache_len, max_age_seconds=max_cache_age)
@@ -85,16 +87,13 @@ class RemoteDatabase:
         req_limit = self.cache.get(api_key)
         if req_limit is None:
             try:
-                r = requests.post(self.url, data={'api_key': api_key})
+                r = requests.post(self.url, data={'api_key': api_key}, timeout=60)
                 res = r.json()
             except Exception as e:
                 print("Cannot authenticate API key: " + str(e))
                 return None
 
-            if res.get('error', None) is None:
+            req_limit = res.get('req_limit', None) if res.get('error', None) is None else None
-                req_limit = res.get('req_limit', None)
-            else:
-                req_limit = None
             self.cache[api_key] = req_limit
 
         return req_limit

libretranslate/app.py

@@ -1,30 +1,37 @@
 import io
 import os
-import tempfile
 import re
+import tempfile
 import uuid
+from datetime import datetime
 from functools import wraps
 from html import unescape
 from timeit import default_timer
-from datetime import datetime
 
 import argostranslatefiles
 from argostranslatefiles import get_supported_formats
-from flask import (abort, Blueprint, Flask, jsonify, render_template, request,
+from flask import Blueprint, Flask, Response, abort, jsonify, render_template, request, send_file, session, url_for
-                   Response, send_file, url_for, session)
+from flask_babel import Babel
+from flask_session import Session
 from flask_swagger import swagger
 from flask_swagger_ui import get_swaggerui_blueprint
-from flask_session import Session
 from translatehtml import translate_html
-from werkzeug.utils import secure_filename
 from werkzeug.exceptions import HTTPException
 from werkzeug.http import http_date
-from flask_babel import Babel
+from werkzeug.utils import secure_filename
 
-from libretranslate import scheduler, flood, secret, remove_translated_files, security, storage
+from libretranslate import flood, remove_translated_files, scheduler, secret, security, storage
 from libretranslate.language import detect_languages, improve_translation_formatting
-from libretranslate.locales import (_, _lazy, get_available_locales, get_available_locale_codes, gettext_escaped,
+from libretranslate.locales import (
-        gettext_html, lazy_swag, get_alternate_locale_links)
+    _,
+    _lazy,
+    get_alternate_locale_links,
+    get_available_locale_codes,
+    get_available_locales,
+    gettext_escaped,
+    gettext_html,
+    lazy_swag,
+)
 
 from .api_keys import Database, RemoteDatabase
 from .suggestions import Database as SuggestionsDatabase
@@ -122,8 +129,8 @@ def create_app(args):
 
     from libretranslate.language import load_languages
 
-    SWAGGER_URL = args.url_prefix + "/docs"  # Swagger UI (w/o trailing '/')
+    swagger_url = args.url_prefix + "/docs"  # Swagger UI (w/o trailing '/')
-    API_URL = args.url_prefix + "/spec"
+    api_url = args.url_prefix + "/spec"
 
     bp = Blueprint('Main app', __name__)
 
@@ -150,10 +157,7 @@ def create_app(args):
         frontend_argos_language_source = languages[0]
 
 
-    if len(languages) >= 2:
+    language_target_fallback = languages[1] if len(languages) >= 2 else languages[0]
-        language_target_fallback = languages[1]
-    else:
-        language_target_fallback = languages[0]
 
     if args.frontend_language_target == "locale":
       def resolve_language_locale():
@@ -185,10 +189,7 @@ def create_app(args):
     if args.req_limit > 0 or args.api_keys or args.daily_req_limit > 0:
         api_keys_db = None
         if args.api_keys:
-            if args.api_keys_remote:
+            api_keys_db = RemoteDatabase(args.api_keys_remote) if args.api_keys_remote else Database(args.api_keys_db_path)
-                api_keys_db = RemoteDatabase(args.api_keys_remote)
-            else:
-                api_keys_db = Database(args.api_keys_db_path)
 
         from flask_limiter import Limiter
 
@@ -220,7 +221,7 @@ def create_app(args):
             os.mkdir(default_mp_dir)
           os.environ["PROMETHEUS_MULTIPROC_DIR"] = default_mp_dir
 
-      from prometheus_client import CONTENT_TYPE_LATEST, Summary, Gauge, CollectorRegistry, multiprocess, generate_latest
+      from prometheus_client import CONTENT_TYPE_LATEST, CollectorRegistry, Gauge, Summary, generate_latest, multiprocess
 
       @bp.route("/metrics")
       @limiter.exempt
@@ -338,7 +339,7 @@ def create_app(args):
             get_api_key_link=args.get_api_key_link,
             web_version=os.environ.get("LT_WEB") is not None,
             version=get_version(),
-            swagger_url=SWAGGER_URL,
+            swagger_url=swagger_url,
             available_locales=[{'code': l['code'], 'name': _lazy(l['name'])} for l in get_available_locales(not args.debug)],
             current_locale=get_locale(),
             alternate_locales=get_alternate_locale_links()
@@ -544,10 +545,7 @@ def create_app(args):
                 )
 
         if args.char_limit != -1:
-            if batch:
+            chars = sum([len(text) for text in q]) if batch else len(q)
-                chars = sum([len(text) for text in q])
-            else:
-                chars = len(q)
 
             if args.char_limit < chars:
                 abort(
@@ -557,10 +555,7 @@ def create_app(args):
 
         if source_lang == "auto":
             source_langs = []
-            if batch:
+            auto_detect_texts = q if batch else [q]
-                auto_detect_texts = q
-            else:
-                auto_detect_texts = [q]
 
             overall_candidates = detect_languages(q)
 
@@ -797,7 +792,7 @@ def create_app(args):
             checked_filepath = security.path_traversal_check(filepath, get_upload_dir())
             if os.path.isfile(checked_filepath):
                 filepath = checked_filepath
-        except security.SuspiciousFileOperation:
+        except security.SuspiciousFileOperationError:
             abort(400, description=_("Invalid filename"))
 
         return_data = io.BytesIO()
@@ -1080,7 +1075,7 @@ def create_app(args):
     swag["info"]["version"] = get_version()
     swag["info"]["title"] = "LibreTranslate"
 
-    @app.route(API_URL)
+    @app.route(api_url)
     @limiter.exempt
     def spec():
         return jsonify(lazy_swag(swag))
@@ -1093,14 +1088,14 @@ def create_app(args):
             return override_lang
         return session.get('preferred_lang', request.accept_languages.best_match(get_available_locale_codes()))
 
-    babel = Babel(app, locale_selector=get_locale)
+    Babel(app, locale_selector=get_locale)
 
     app.jinja_env.globals.update(_e=gettext_escaped, _h=gettext_html)
 
     # Call factory function to create our blueprint
-    swaggerui_blueprint = get_swaggerui_blueprint(SWAGGER_URL, API_URL)
+    swaggerui_blueprint = get_swaggerui_blueprint(swagger_url, api_url)
     if args.url_prefix:
-        app.register_blueprint(swaggerui_blueprint, url_prefix=SWAGGER_URL)
+        app.register_blueprint(swaggerui_blueprint, url_prefix=swagger_url)
     else:
         app.register_blueprint(swaggerui_blueprint)
 

libretranslate/detect.py

@@ -2,10 +2,11 @@
 
 import pycld2 as cld2
 
-class UnknownLanguage(Exception):
+
+class UnknownLanguageError(Exception):
   pass
 
-class Language(object):
+class Language:
   def __init__(self, choice):
     name, code, confidence, bytesize = choice
     self.code = code
@@ -23,7 +24,7 @@ class Language(object):
     return Language(("", code, 100, 0))
 
 
-class Detector(object):
+class Detector:
   """ Detect the language used in a snippet of text."""
 
   def __init__(self, text, quiet=False):
@@ -56,17 +57,16 @@ class Detector(object):
     if not reliable:
       self.reliable = False
       reliable, index, top_3_choices = cld2.detect(text, bestEffort=True)
-      
+
-      if not self.quiet:
+      if not self.quiet and not reliable:
-        if not reliable:
+        raise UnknownLanguageError("Try passing a longer snippet of text")
-          raise UnknownLanguage("Try passing a longer snippet of text")
 
     self.languages = [Language(x) for x in top_3_choices]
     self.language = self.languages[0]
     return self.language
 
   def __str__(self):
-    text = "Prediction is reliable: {}\n".format(self.reliable)
+    text = f"Prediction is reliable: {self.reliable}\n"
-    text += u"\n".join(["Language {}: {}".format(i+1, str(l))
+    text += "\n".join([f"Language {i+1}: {str(l)}"
                         for i,l in enumerate(self.languages)])
-    return text
+    return text

libretranslate/init.py

@@ -1,4 +1,3 @@
-from pathlib import Path
 
 from argostranslate import package, translate
 
@@ -46,14 +45,12 @@ def check_and_install_models(force=False, load_only_lang_codes=None):
         # Download and install all available packages
         for available_package in available_packages:
             print(
-                "Downloading %s (%s) ..."
+                f"Downloading {available_package} ({available_package.package_version}) ..."
-                % (available_package, available_package.package_version)
             )
             available_package.install()
 
         # reload installed languages
         libretranslate.language.languages = translate.get_installed_languages()
         print(
-            "Loaded support for %s languages (%s models total)!"
+            f"Loaded support for {len(translate.get_installed_languages())} languages ({len(available_packages)} models total)!"
-            % (len(translate.get_installed_languages()), len(available_packages))
         )

libretranslate/language.py

@@ -1,7 +1,7 @@
-import string
 
 from argostranslate import translate
-from libretranslate.detect import Detector, UnknownLanguage
+
+from libretranslate.detect import Detector, UnknownLanguageError
 
 __languages = None
 
@@ -29,7 +29,7 @@ def detect_languages(text):
             for i in range(len(d)):
                 d[i].text_length = len(t)
             candidates.extend(d)
-        except UnknownLanguage:
+        except UnknownLanguageError:
             pass
 
     # total read bytes of the provided text
@@ -83,10 +83,10 @@ def improve_translation_formatting(source, translation, improve_punctuation=True
 
     if not len(source):
         return ""
-    
+
     if not len(translation):
         return source
-    
+
     if improve_punctuation:
         source_last_char = source[len(source) - 1]
         translation_last_char = translation[len(translation) - 1]

libretranslate/locales.py

@@ -1,10 +1,11 @@
-import os
 import json
+import os
 from functools import lru_cache
+
 from flask_babel import gettext as _
 from flask_babel import lazy_gettext as _lazy
+from markupsafe import Markup, escape
 
-from markupsafe import escape, Markup
 
 @lru_cache(maxsize=None)
 def get_available_locales(only_reviewed=True, sort_by_name=False):

libretranslate/main.py

@@ -197,7 +197,7 @@ def main():
             from waitress import serve
 
             url_scheme = "https" if args.ssl else "http"
-            print("Running on %s://%s:%s%s" % (url_scheme, args.host, args.port, args.url_prefix))
+            print(f"Running on {url_scheme}://{args.host}:{args.port}{args.url_prefix}")
 
             serve(
                 app,

libretranslate/manage.py

@@ -49,7 +49,7 @@ def manage():
                 print("There are no API keys")
             else:
                 for item in keys:
-                    print("%s: %s" % item)
+                    print("{}: {}".format(*item))
 
         elif args.sub_command == "add":
             print(db.add(args.req_limit, args.key)[0])

libretranslate/scheduler.py

@@ -1,5 +1,7 @@
 import atexit
+
 from apscheduler.schedulers.background import BackgroundScheduler
+
 scheduler = None
 
 def setup(args):

libretranslate/secret.py

@@ -1,9 +1,9 @@
-import atexit
 import random
 import string
 
 from libretranslate.storage import get_storage
 
+
 def generate_secret():
     return ''.join(random.choices(string.ascii_uppercase + string.digits, k=7))
 

libretranslate/security.py

@@ -1,7 +1,7 @@
 import os
 
 
-class SuspiciousFileOperation(Exception):
+class SuspiciousFileOperationError(Exception):
     pass
 
 
@@ -10,7 +10,7 @@ def path_traversal_check(unsafe_path, known_safe_path):
     unsafe_path = os.path.abspath(unsafe_path)
 
     if (os.path.commonprefix([known_safe_path, unsafe_path]) != known_safe_path):
-        raise SuspiciousFileOperation("{} is not safe".format(unsafe_path))
+        raise SuspiciousFileOperationError(f"{unsafe_path} is not safe")
 
     # Passes the check
-    return unsafe_path
+    return unsafe_path

libretranslate/suggestions.py

@@ -1,5 +1,5 @@
-import sqlite3
 import os
+import sqlite3
 
 from expiringdict import ExpiringDict
 
@@ -10,7 +10,7 @@ class Database:
     def __init__(self, db_path=DEFAULT_DB_PATH, max_cache_len=1000, max_cache_age=30):
         # Legacy check - this can be removed at some point in the near future
         if os.path.isfile("suggestions.db") and not os.path.isfile("db/suggestions.db"):
-            print("Migrating %s to %s" % ("suggestions.db", "db/suggestions.db"))
+            print("Migrating {} to {}".format("suggestions.db", "db/suggestions.db"))
             try:
                 os.rename("suggestions.db", "db/suggestions.db")
             except Exception as e:

libretranslate/tests/test_api/conftest.py

@@ -1,4 +1,5 @@
 import sys
+
 import pytest
 
 from libretranslate.app import create_app

libretranslate/tests/test_api/test_api_translate.py

@@ -43,7 +43,7 @@ def test_api_translate_unsupported_language(client):
     response_json = json.loads(response.data)
 
     assert "error" in response_json
-    assert "zz is not supported" == response_json["error"]
+    assert response_json["error"] == "zz is not supported"
     assert response.status_code == 400
 
 
@@ -57,5 +57,5 @@ def test_api_translate_missing_parameter(client):
     response_json = json.loads(response.data)
 
     assert "error" in response_json
-    assert "Invalid request: missing q parameter" == response_json["error"]
+    assert response_json["error"] == "Invalid request: missing q parameter"
     assert response.status_code == 400

libretranslate/tests/test_init.py

@@ -1,6 +1,7 @@
-from libretranslate.init import boot
 from argostranslate import package
 
+from libretranslate.init import boot
+
 
 def test_boot_argos():
     """Test Argos translate models initialization"""

pyproject.toml

@@ -0,0 +1,167 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+requires-python = ">=3.8"
+name = "libretranslate"
+description = "Free and Open Source Machine Translation API. Self-hosted, no limits, no ties to proprietary services."
+readme = "README.md"
+license = { file = "LICENSE" }
+authors = [
+    { name = "Piero Toffanin", email = "pt@uav4geo.com" },
+    { name = "LibreTranslate Authors" },
+]
+maintainers = [
+    { name = "Piero Toffanin", email = "pt@uav4geo.com" },
+    { name = "LibreTranslate Authors" },
+]
+keywords = [
+    "Python",
+    "Translate",
+    "Translation",
+    "API",
+]
+classifiers = [
+    "Operating System :: OS Independent",
+    "License :: OSI Approved :: GNU Affero General Public License v3",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10"
+]
+dynamic = ["version"]
+
+dependencies = [
+    "argostranslate ==1.8.0",
+    "Flask ==2.2.2",
+    "flask-swagger ==0.2.14",
+    "flask-swagger-ui ==4.11.1",
+    "Flask-Limiter ==2.6.3",
+    "Flask-Babel ==3.1.0",
+    "Flask-Session ==0.4.0",
+    "waitress ==2.1.2",
+    "expiringdict ==1.2.2",
+    " LTpycld2==0.42",
+    "morfessor ==2.0.6",
+    "appdirs ==1.4.4",
+    "APScheduler ==3.9.1",
+    "translatehtml ==1.5.2",
+    "argos-translate-files ==1.1.1",
+    "itsdangerous ==2.1.2",
+    "Werkzeug ==2.2.2",
+    "requests ==2.28.1",
+    "redis ==4.3.4",
+    "prometheus-client ==0.15.0",
+    "polib ==1.1.1",
+]
+
+[project.scripts]
+libretranslate = "libretranslate.main:main"
+ltmanage = "libretranslate.manage:manage"
+
+
+[project.optional-dependencies]
+test = [
+    "pytest >=7.2.0",
+    "pytest-cov",
+    "ruff ==0.0.277",
+    "types-requests",
+]
+
+
+[project.urls]
+Homepage = "https://libretranslate.com"
+Source = "https://github.com/LibreTranslate/LibreTranslate"
+Documentation = "https://github.com/LibreTranslate/LibreTranslate"
+Tracker = "https://github.com/LibreTranslate/LibreTranslate/issues"
+History = "https://github.com/LibreTranslate/LibreTranslate/releases"
+
+
+# ENVIRONMENTS AND SCRIPTS
+[tool.hatch.envs.default]
+features = [
+    "test",
+]
+
+
+[tool.hatch.envs.default.scripts]
+dev = "python main.py {args}"
+locales = "python scripts/compile_locales.py"
+fmt = [
+    "ruff libretranslate scripts --fix",
+]
+test = [
+    "fmt",
+    "pytest {args}",
+]
+cov = [
+    "pytest --cov-report html {args}",
+    "python -c 'import webbrowser; webbrowser.open(\"http://0.0.0.0:3000\")'",
+    "python -m http.server 3000 --directory ./htmlcov",
+]
+
+
+[[tool.hatch.envs.all.matrix]]
+python = ["3.8", "3.9", "3.10", "3.11"]
+
+
+# TOOLS
+[tool.hatch.version]
+path = "VERSION"
+pattern = "^(?P<version>[0-9]*.[0-9]*.[0-9]*)$"
+
+
+[tool.pytest.ini_options]
+addopts = [
+    "-v",
+    "--cov=libretranslate",
+    "--color=yes",
+    "--cov-report=term-missing",
+]
+
+
+# https://beta.ruff.rs/docs/rules
+[tool.ruff]
+src = ["libretranslate", "scripts"]
+target-version = "py38"
+line-length = 136
+select = [
+    "I",     # isort
+    "N",     # pep8-naming
+    "S",     # bandit
+    "A",     # flake8-builtins
+    "YTT",   # flake8-2020
+    "B",     # flake8-bugbear
+    # "C",     # flake8-comprehensions
+    "ICN",   # flake8-import-conventions
+    # "SIM",   # flake8-simplify
+    "TID",   # flake8-tidy-imports
+    # "Q",     # flake8-quotes
+    # "FBT",   # flake8-boolean-trap
+    "F",     # pyflakes
+    "UP",    # pyupgrade
+    # "E",     # pycodestyle errors
+    # "W",     # pycodestyle warnings
+    # "PLC",   # pylint convention
+    "PLE",   # pylint error
+    # "PLR",   # pylint refactor
+    # "PLW",   # pylint warning
+    # "RUF",   # ruff specific
+]
+
+ignore = [
+    "E501", # line too long
+    "A003", # Class attribute is shadowing a python builtin
+    "S101", # Use of `assert` detected
+    "S311", # Standard pseudo-random generators are not suitable for cryptographic purposes
+    "T201", "T203", # remove print and pprint
+    "E402", # Module level import not at top of file
+]
+
+[tool.ruff.per-file-ignores]
+"__init__.py" = ["I", "F401"] # module imported but unused
+
+
+[tool.ruff.mccabe]
+max-complexity = 12

requirements.txt

@@ -1,21 +0,0 @@
-argostranslate==1.8.0
-Flask==2.2.2
-flask-swagger==0.2.14
-flask-swagger-ui==4.11.1
-Flask-Limiter==2.6.3
-Flask-Babel==3.1.0
-Flask-Session==0.4.0
-waitress==2.1.2
-expiringdict==1.2.2
-LTpycld2==0.42
-morfessor==2.0.6
-appdirs==1.4.4
-APScheduler==3.9.1
-translatehtml==1.5.2
-argos-translate-files==1.1.1
-itsdangerous==2.1.2
-Werkzeug==2.2.2
-requests==2.28.1
-redis==4.3.4
-prometheus-client==0.15.0
-polib==1.1.1

scripts/compile_locales.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
-import sys
 import os
+import sys
+
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 from babel.messages.frontend import main as pybabel
 
@@ -15,7 +16,7 @@ if __name__ == "__main__":
             link = "https://hosted.weblate.org/translate/libretranslate/app/%s/" % l['code']
             if l['code'] == 'en':
                 link = "https://hosted.weblate.org/projects/libretranslate/app/"
-            print("%s | %s | %s" % (l['name'], ':heavy_check_mark:' if l['reviewed'] else '', "[Edit](%s)" % link))
+            print("{} | {} | {}".format(l['name'], ':heavy_check_mark:' if l['reviewed'] else '', "[Edit](%s)" % link))
     else:
         locales_dir = os.path.join("libretranslate", "locales")
         if not os.path.isdir(locales_dir):

scripts/gunicorn_conf.py

@@ -1,7 +1,9 @@
-from prometheus_client import multiprocess
 import re
 import sys
 
+from prometheus_client import multiprocess
+
+
 def child_exit(server, worker):
     multiprocess.mark_process_dead(worker.pid)
 
@@ -35,7 +37,7 @@ def on_starting(server):
 
     args = get_args()
     
-    from libretranslate import storage, scheduler, flood, secret
+    from libretranslate import flood, scheduler, secret, storage
     storage.setup(args.shared_storage)
     scheduler.setup(args)
     flood.setup(args)

scripts/healthcheck.py

@@ -1,4 +1,5 @@
 import requests
+
 response = requests.post(
     url='http://0.0.0.0:5000/translate',
     headers={'Content-Type': 'application/json'},
@@ -6,6 +7,7 @@ response = requests.post(
          'q': 'Hello World!',
          'source': 'en',
          'target': 'en'
-    }
+    },
+    timeout=60
 )
 # if server unavailable then requests with raise exception and healthcheck will fail

scripts/install_models.py

@@ -1,8 +1,10 @@
 #!/usr/bin/env python
-import sys
 import os
+import sys
+
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 import argparse
+
 from libretranslate.init import check_and_install_models
 
 if __name__ == "__main__":

scripts/suggestions-to-jsonl.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python
 import argparse
-import time
-import sqlite3
 import json
+import sqlite3
+import time
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Program to generate JSONL files from a LibreTranslate's suggestions.db")

scripts/update_locales.py

@@ -1,17 +1,19 @@
 #!/usr/bin/env python
-import sys
 import os
+import sys
+
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
-import re
-import polib
 import json
+import re
+
+import polib
 from babel.messages.frontend import main as pybabel
-from libretranslate.language import load_languages, improve_translation_formatting
-from libretranslate.locales import get_available_locale_codes, swag_eval
-from translatehtml import translate_html
-from libretranslate.app import get_version, create_app
-from libretranslate.main import get_args
 from flask_swagger import swagger
+from libretranslate.app import create_app, get_version
+from libretranslate.language import improve_translation_formatting, load_languages
+from libretranslate.locales import get_available_locale_codes, swag_eval
+from libretranslate.main import get_args
+from translatehtml import translate_html
 
 # Update strings
 if __name__ == "__main__":
@@ -74,7 +76,7 @@ if __name__ == "__main__":
         if not os.path.isfile(meta_file):
             with open(meta_file, 'w') as f:
                 f.write(json.dumps({
-                    'name': next((lang.name for lang in languages if lang.code == l)),
+                    'name': next(lang.name for lang in languages if lang.code == l),
                     'reviewed': False
                 }, indent=4))
                 print("Wrote %s" % meta_file)

setup.cfg

@@ -1,12 +0,0 @@
-[flake8]
-exclude = .git,
-        .vscode,
-        .gitignore,
-        README.md,
-        venv,
-        test,
-        setup.py,
-        libretranslate/__init__.py
-
-max-line-length = 136
-ignore = E741

setup.py

@@ -1,38 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-from setuptools import setup, find_packages
-
-setup(
-    version=open('VERSION').read().strip(),
-    name='libretranslate',
-    license='GNU Affero General Public License v3.0',
-    description='Free and Open Source Machine Translation API. Self-hosted, no limits, no ties to proprietary services.',
-    author='LibreTranslate Authors',
-    author_email='pt@uav4geo.com',
-    url='https://libretranslate.com',
-    packages=find_packages(),
-    # packages=find_packages(include=['openpredict']),
-    # package_dir={'openpredict': 'openpredict'},
-    package_data={'': ['static/*', 'static/**/*', 'templates/*', 'locales/**/meta.json', 'locales/**/**/*.mo']},
-    include_package_data=True,
-    entry_points={
-        'console_scripts': [
-            'libretranslate=libretranslate.main:main',
-            'ltmanage=libretranslate.manage:manage'
-        ],
-    },
-
-    python_requires='>=3.8.0',
-    long_description=open('README.md').read(),
-    long_description_content_type="text/markdown",
-    install_requires=open("requirements.txt", "r").readlines(),
-    tests_require=['pytest==7.2.0'],
-    setup_requires=['pytest-runner'],
-    classifiers=[
-        "License :: OSI Approved :: GNU Affero General Public License v3 ",
-        "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-        "Programming Language :: Python :: 3.10"
-    ]
-)