U nld@sddlZddlZddlZddlZddlZddlmZddlmZddl m Z ddl m Z ddl Z ddl m Z ddlmZmZmZmZmZmZmZmZmZmZddlmZdd lmZdd lmZdd lm Z dd l!m"Z"dd l#m$Z$ddl%m&Z&ddl'm(Z(ddl)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/ddl0m1Z1m2Z2ddl3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;ddlZ>ddl?m=Z@ddZAddZBddZCddZDddZEd d!ZFd(d"d#ZGd$d%ZHd&d'ZIdS))N)wraps)unescape) default_timer)datetime)get_supported_formats) abort BlueprintFlaskjsonifyrender_templaterequestResponse send_fileurl_forsession)swagger)get_swaggerui_blueprint)Session)translate_html)secure_filename) HTTPException) http_date)Babel) schedulerfloodsecretremove_translated_filessecuritystorage)detect_languagesimprove_translation_formatting)__lazyget_available_localesget_available_locale_codesgettext_escaped gettext_html lazy_swagget_alternate_locale_links)DatabaseRemoteDatabase)r*c CsBz.td}|W5QRWSQRXWnYdSXdS)NZVERSION?)openreadstrip)fr12/var/www/html/LibreTranslate/libretranslate/app.py get_version!s  $r3cCs,tjtd}tj|s(t||S)Nzlibretranslate-files-translate)ospathjointempfileZ gettempdirisdirmkdir)Z upload_dirr1r1r2get_upload_dir)s  r:cCs*tjrtt}|d}n tjd}|S)Napi_keyr is_json get_json_dictgetvaluesjsonakr1r1r2get_req_api_key2s   rDcCs*tjrtt}|d}n tjd}|S)Nrr<rAr1r1r2get_req_secret;s   rEcCs&|}t|ts"tdtdd|S)NzInvalid JSON format description)Zget_json isinstancedictrr!)r dr1r1r2r>Es r>cCs6tjdr(tjdddd}n tjp0d}|S)NzX-Forwarded-Forr, 127.0.0.1)r headersZgetlistsplitZ remote_addr)ipr1r1r2get_remote_addressLs  rQcCs0|}|r,t}|r,||}|dk r,||}|SN)rDlookup)Z default_limit api_keys_dbZ multiplier req_limitr;Z db_req_limitr1r1r2get_req_limitsUs rVcsDdkr dfdd}fdd}|g}dkr@|||S)Nla$csdtS)Nz %s per minuterVr1)rTdefault_req_limitr1r2 minute_limitshsz(get_routes_limits..minute_limitscsdtdS)Nz %s per dayirXr1)rTdaily_req_limitr1r2 daily_limitsksz'get_routes_limits..daily_limitsr)append)rYr[rTrZr\resr1)rTr[rYr2get_routes_limitscs r_c"sddlm}|jjddlm}jdjd}tdt}t j j sbt t|i D]}tdd|jD |j<qpjd krtd tfd td d nttfd dDddkrdtdkrd nd jdkr fdd}|n6ttfddDd dkrJ fddgtD]}|jD]}|qjq`djdksjsjdkrdjrΈj rt!j n t"j#ddl$m%} | t&t'jjj(d} nddl)m%} | } dt*j+,ddkr(t- t. t/ ddj0r"t*j+,ddkrt*j12t*j13dd} t*j14| st*5| | t*j+d<ddl6m7m8} m9} m:m;m< |=d| j> fd d!}| d"d#d$d%d&d'g?d(d)d*d| d+d,d$d&d'gd-d.?d(d*dfd/d0}|@d1d2d3}|@d4d5d6}|@d7d8d9}|@d:d;d<}|=d=| j> fd>d?}|=d@| j>fdAdB}|,dC| j> fdDdE}|jAdFdG}|Bd(|fdHdI}|BdJ|fdKdL}|,dMtCdNfdOdP }|BdQ|dRdS}|=dT| j>fdUdV}|BdW|fdXdY}tDt}dZ|jEd[<t*j13dd\|jEd]<tF|jGrd^|jEd_<jr|jH|jd`n |H|| I|tJ|tKdadb<dcdadd<|=|| j>fdedf}dg|jEdh<didj tL| dk} |jMjNjOtPtQdltR|}!jr|jH|!d`n |H|!|S)mNr)boot)load_languagesz/docsz/speczMain appcSsg|] }|jjqSr1)Zto_langcode.0lr1r1r2 szcreate_app..autoobjz Auto Detectrbnamecsg|]}|jjkr|qSr1)rbfrontend_language_sourcercargsr1r2rfs r)localecs2ttfddDd}|dkr.}|S)Ncsg|]}|jkr|qSr1rbrclocr1r2rfs z?create_app..resolve_language_locale..nextiterlanguage_target) get_localelanguage_target_fallback languagesrqr2resolve_language_localesz+create_app..resolve_language_localecsg|]}|jjkr|qSr1)rbfrontend_language_targetrcrlr1r2rfs csSrRr1r1rvr1r2zcreate_app..)Limiter)Zkey_funcZdefault_limitsZ storage_uriZgunicornZSERVER_SOFTWAREZPROMETHEUS_MULTIPROC_DIRZdbZ prometheus)CONTENT_TYPE_LATESTSummaryGaugeCollectorRegistry multiprocessgenerate_latestz/metricscsPjr0tjd}|djkr0tdtdd}|t|dS)N AuthorizationzBearer iZ UnauthorizedrG)Zmimetype)Zmetrics_auth_tokenr rNr?rr!ZMultiProcessCollectorr )Z authorizationregistry)rrrmrrr1r2prometheus_metricss  z&create_app..prometheus_metricsZ,libretranslate_http_request_duration_secondszTime spent on requestZendpointstatusZ request_ipr;z /translaterMZ&libretranslate_http_requests_in_flightzActive requestsZlivesum)Zmultiprocess_modecsBtfddjr:tfdd}|SSdS)Ncst}t|r tdtddjrt}|rP|dkrPtdtddn~d}|dk}jr|rt jt j ddsd}jr|rttsd}|rtd }jrtd jd }td |d||S) Nz"Too many request limits violationsrGzInvalid API keyFZOriginrTz4Please contact the server operator to get an API keyzVisit %(url)s to get an API key)ZurlrF)rQrZ is_bannedrr!api_keysrDrSrequire_api_key_originrematchr rNr?require_api_key_secretrZ secret_matchrEget_api_key_link)akwrPrCZneed_keyZ key_missingrH)rTrmr0r1r2funcsB  z.create_app..access_check..funcc st}d}t}tpd}tj||}zJz| ||WW2St k rp}z|j }|W5d}~XYnXW5tt|d}tj|||||XdS)Nrrr) rrQrDlabelsr r5maxZobserveZdecZincrrb) rrZstart_trrPrCgZduratione)r gauge_requestmeasure_requestr1r2 measure_funcs z6create_app..access_check..measure_func)rmetrics)r0r)rTrmrr)r0rr2 access_checks'z create_app..access_checkrFcSstdt|jidfS)NerrorrFr strrHrr1r1r2 invalid_api2szcreate_app..invalid_apicSstdt|jidfS)Nrrrrr1r1r2 server_error6sz create_app..server_errorcSs.tttdtddt|jidfS)Nrz Slowdown: r)rreportrQr r!rrHrr1r1r2slow_down_error:s z#create_app..slow_down_errorrcSstdt|jidfS)Nrrrrr1r1r2denied?szcreate_app..denied/c sjrtdtjd}|r:|tj kr:tj|dt dj j j j tjddk tddtj Dtd S) Nlang)preferred_langz index.htmlZLT_WEBcSs"g|]}|dt|ddqS)rbrjri)r"rcr1r1r2rfVsz-create_app..index..) ZgaIdfrontendTimeoutrrZ web_versionversionZ swagger_urlZavailable_localesZcurrent_localeZalternate_locales)disable_web_uirr rmr?r$debugrupdater Zga_idfrontend_timeoutrrr4environr3r#r()Zlangcode) SWAGGER_URLrmrxr1r2indexCs$  zcreate_app..indexz /js/app.jscsrjrtdttdjjjr*tndddd}jrnt t |j d<d|j d <d |j d <d |j d <|S)Nrzapp.js.templater) url_prefixrZ api_secretz%application/javascript; charset=utf-8)Z content_typez Last-ModifiedzIno-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0z Cache-Controlzno-cacheZPragmaz-1ZExpires) rrr r rrrrZget_current_secretrrZnowrNZresponserlr1r2appjs[s   zcreate_app..appjsz /languagescstfddDS)a Retrieve list of supported languages --- tags: - translate responses: 200: description: List of languages schema: id: languages type: array items: type: object properties: code: type: string description: Language code name: type: string description: Human-readable language name (in English) targets: type: array items: type: string description: Supported target language codes cs*g|]"}|jt|j|jgdqS))rbrjZtargets)rbr"rjr?rc)language_pairsr1r2rfsz-create_app..langs..)r r1)rrzr1r2langsnszcreate_app..langscSsX|jdd|jdd|jdd|jdd|jd d |jd d |S) NzAccess-Control-Allow-Origin*zAccess-Control-Allow-HeaderszAuthorization, Content-TypezAccess-Control-Expose-HeadersrzAccess-Control-Allow-Methodsz GET, POSTz Access-Control-Allow-CredentialstruezAccess-Control-Max-Agei^)rNaddrr1r1r2 after_requestsz!create_app..after_requestc sTtjr8tt}|d}|d|d|d}n0tjd}tjdtjdtjd}|stdtddddstdtddddstdtddddt|t}|rj d krt |}j |krtdtd |j d dj d krD|rt d d |D}nt |}j |krDtdtd |j d ddkrg}|r^|}n|g}t |}|D]V} t | dkrt | } n|} || djrpt| | td| ddqpn&|rfdd |D}n ddg}fdd |D} t| D],\} } | dkr tdtd|| ddq ttfdd Dd}|dkrrtdtddd|s|d}|dkrtdtd|ddzv|rhg}t|D]\} }| | |}|dkrtdtdt|j|jt| | j| | jd d|d!krtt||}nt|||}|t|qdkrXt||d"WStd#|iWSn| d|}|dkrtdtdt|j|jt| dj| djd d|d!krtt||}nt|||}dkrtt||dd"WStd#t|iWSWn<tk rN}ztd$td%t|d&dW5d}~XYnXdS)'a Translate text from a language to another --- tags: - translate parameters: - in: formData name: q schema: oneOf: - type: string example: Hello world! - type: array example: ['Hello world!'] required: true description: Text(s) to translate - in: formData name: source schema: type: string example: en required: true description: Source language code - in: formData name: target schema: type: string example: es required: true description: Target language code - in: formData name: format schema: type: string enum: [text, html] default: text example: text required: false description: > Format of source text: * `text` - Plain text * `html` - HTML markup - in: formData name: api_key schema: type: string example: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx required: false description: API key responses: 200: description: Translated text schema: id: translate type: object properties: translatedText: oneOf: - type: string - type: array description: Translated text(s) 400: description: Invalid request schema: id: error-response type: object properties: error: type: string description: Error message 500: description: Translation error schema: id: error-response type: object properties: error: type: string description: Error message 429: description: Slow down schema: id: error-slow-down type: object properties: error: type: string description: Reason for slow down 403: description: Banned schema: id: error-response type: object properties: error: type: string description: Error message qsourcetargetformatrF+Invalid request: missing %(name)s parameterrjrGrWzBInvalid request: request (%(size)s) exceeds text limit (%(limit)s))sizeZlimitcSsg|] }t|qSr1)lenrdtextr1r1r2rfsz1create_app..translate..rg(rzAuto detected: %slanguagecsg|]}ddqS)Y@Z confidencerr1r source_langr1r2rf>srrcs(g|] ttfddDdqS)csg|]}|jdkr|qS)rrprcrr1r2rfBsz.translate...Nrsrdrzrr2rfBsN%(lang)s is not supportedrcsg|]}|jkr|qSr1rprc target_langr1r2rfHs r)rhtmlz"%(format)s format is not supported)rzV%(tname)s (%(tcode)s) is not available as a target language from %(sname)s (%(scode)s))ZtnameZtcodeZsnameZscoder)translatedTextZdetectedLanguagerrzCannot translate text: %(text)s)r)r r=r>r?r@rr!rIlistZ batch_limitr char_limitsumrr]rprint enumeratertruget_translationr"rjrbrrr translaterr Exception)rBrZ text_formatZbatchZ batch_sizechars source_langsZauto_detect_textsZoverall_candidatesZ text_to_checkZcandidate_langs src_langsidxrtgt_langZresultsrZ translatorZtranslated_textr)rmrz)rrr2rse                    2   2   zcreate_app..translatez/translate_filec sjrtdtddtjd}tjdtjd}|sPtdtddd d|shtdtddd dstdtddd d|jd krtdtd dtj |jd krtdtd d|g}fdd|D}t |D](\}}|dkrtdtd||ddqt t fddDd}|dkrDtdtdddznttdt|j}tj t|}||t|d||} tj | } tdtd| ddiWStk r} ztd| dW5d} ~ XYnXdS)a Translate file from a language to another --- tags: - translate consumes: - multipart/form-data parameters: - in: formData name: file type: file required: true description: File to translate - in: formData name: source schema: type: string example: en required: true description: Source language code - in: formData name: target schema: type: string example: es required: true description: Target language code - in: formData name: api_key schema: type: string example: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx required: false description: API key responses: 200: description: Translated file schema: id: translate-file type: object properties: translatedFileUrl: type: string description: Translated file url 400: description: Invalid request schema: id: error-response type: object properties: error: type: string description: Error message 500: description: Translation error schema: id: error-response type: object properties: error: type: string description: Error message 429: description: Slow down schema: id: error-slow-down type: object properties: error: type: string description: Reason for slow down 403: description: Banned schema: id: error-response type: object properties: error: type: string description: Error message r.Files translation are disabled on this server.rGrrfilerFrrrzInvalid request: empty filer)z*Invalid request: file format not supportedcs(g|] ttfddDdqS)csg|]}|jkr|qSr1rprcrr1r2rfs zAcreate_app..translate_file...Nrsrrrr2rfsz6create_app..translate_file..Nrrcsg|]}|jkr|qSr1rprcrr1r2rfs .rZtranslatedFileUrlzMain app.download_fileT)filenameZ _externalr)disable_files_translationrr!r Zformr?filesrr4r5splitextrrtruruuidZuuid4rr6r:Zsaveargostranslatefilestranslate_filerbasenamer rr) rrrrrrrrfilepathZtranslated_file_pathZtranslated_filenamer)rm%frontend_argos_supported_files_formatrzrr2rsHT        z"create_app..translate_filez /download_file/)rc sjrtdtddtjt|}z"t|t}tj |rF|}Wn&tj k rntdtddYnXt }t |d}||W5QRX|d|d}|dd|}t|d|d S) z, Download a translated file rFrrGzInvalid filenamerbrrT)Z as_attachmentZ download_name)rrr!r4r5r6r:rZpath_traversal_checkisfileZSuspiciousFileOperationioBytesIOr-writer.seekrOpopr)rrZchecked_filepathZ return_datafoZdownload_filenamerlr1r2 download_file s"      z!create_app..download_filez/detectcSsJtjrtt}|d}n tjd}|s>tdtddddtt|S)a Detect the language of a single text --- tags: - translate parameters: - in: formData name: q schema: type: string example: Hello world! required: true description: Text to detect - in: formData name: api_key schema: type: string example: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx required: false description: API key responses: 200: description: Detections schema: id: detections type: array items: type: object properties: confidence: type: number format: float minimum: 0 maximum: 1 description: Confidence value example: 0.6 language: type: string description: Language code example: en 400: description: Invalid request schema: id: error-response type: object properties: error: type: string description: Error message 500: description: Detection error schema: id: error-response type: object properties: error: type: string description: Error message 429: description: Slow down schema: id: error-slow-down type: object properties: error: type: string description: Reason for slow down 403: description: Banned schema: id: error-response type: object properties: error: type: string description: Error message rrFrrrG) r r=r>r?r@rr!r r)rBrr1r1r2detect(sP  zcreate_app..detectz/frontend/settingsc sd}tjjjtjo jjj jr6gnjt j d|jt |j dddS)a Retrieve frontend specific settings --- tags: - frontend responses: 200: description: frontend settings schema: id: frontend-settings type: object properties: charLimit: type: integer description: Character input limit for this language (-1 indicates no limit) frontendTimeout: type: integer description: Frontend translation timeout apiKeys: type: boolean description: Whether the API key database is enabled. keyRequired: type: boolean description: Whether an API key is required. suggestions: type: boolean description: Whether submitting suggestions is enabled. supportedFilesFormat: type: array items: type: string description: Supported files format language: type: object properties: source: type: object properties: code: type: string description: Language code name: type: string description: Human-readable language name (in English) target: type: object properties: code: type: string description: Language code name: type: string description: Human-readable language name (in English) ri)rr)Z charLimitrZapiKeysZ keyRequired suggestionsZfilesTranslationZsupportedFilesFormatr) r rrrboolrrrrbr"rjr)rmfrontend_argos_language_sourcefrontend_argos_language_targetrr1r2frontend_settingss$9 z%create_app..frontend_settingsz/suggestcsjstdtddtjd}tjd}tjd}tjd}|s^tdtd dd d|svtdtd dd d|stdtd dd d|stdtd dd dt||||td d iS) u Submit a suggestion to improve a translation --- tags: - feedback parameters: - in: formData name: q schema: type: string example: Hello world! required: true description: Original text - in: formData name: s schema: type: string example: ¡Hola mundo! required: true description: Suggested translation - in: formData name: source schema: type: string example: en required: true description: Language of original text - in: formData name: target schema: type: string example: es required: true description: Language of suggested translation responses: 200: description: Success schema: id: suggest-response type: object properties: success: type: boolean description: Whether submission was successful 403: description: Not authorized schema: id: error-response type: object properties: error: type: string description: Error message rz(Suggestions are disabled on this server.rGrsrrrFrrZsuccessT) rrr!r r@r?SuggestionsDatabaserr )rrrrrlr1r2suggests 9    zcreate_app..suggestZ filesystemZ SESSION_TYPEZsessionsZSESSION_FILE_DIRTZTEMPLATES_AUTO_RELOAD)rinforZLibreTranslatetitlecs ttSrR)r r'r1)swagr1r2spec4szcreate_app..specZlocalesZBABEL_TRANSLATION_DIRECTORIEScSs4tjd}|r|tkr|StdtjtS)NzX-Override-Accept-Languager)r rNr?r$rZaccept_languagesZ best_match)Z override_langr1r1r2rx;s zcreate_app..get_locale)Zlocale_selector)Z_eZ_h)SZlibretranslate.initr`Z load_onlyZ update_modelslibretranslate.languagerarr__name__rZsetupZshared_storagerrr:sortedZtranslations_fromrbrktypeobjectr!rtrurr|rZsupported_file_extensionsr]rUrr[Zapi_keys_remoter+r*Zapi_keys_db_pathZ flask_limiterrrQr_Zreq_limit_storageZ no_limiterr4rr?rrrrr5abspathr6r8r9Zprometheus_clientrrrrrrZrouteZexemptrZ errorhandlerrZpostrr ZconfigrrZregister_blueprintZinit_apprr3rZ jinja_envglobalsrr%r&r)"rmr`raZAPI_URLZbprr{Z file_formatZffrZlimiterZdefault_mp_dirrrrrrrrrrrrrrrrrrrapprZbabelZswaggerui_blueprintr1)rrrrTrmrrrrrrxrrwryrzrrrr2 create_appvs"                             ?     m YOK         r)r))Jrr4r7rr functoolsrrrZtimeitrrrrZflaskrrr r r r r rrrZ flask_swaggerrZflask_swagger_uirZ flask_sessionrZ translatehtmlrZwerkzeug.utilsrZwerkzeug.exceptionsrZ werkzeug.httprZ flask_babelrlibretranslaterrrrrrr rr Zlibretranslate.localesr!r"r#r$r%r&r'r(rr*r+rrr3r:rDrEr>rQrVr_rr1r1r1r2sB     0         (