-
Notifications
You must be signed in to change notification settings - Fork 6
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #20 from NikolayS/new_apis
New_apis
- Loading branch information
Showing
18 changed files
with
771 additions
and
58 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1 +1,3 @@ | ||
| *.swp | ||
| *~ | ||
| /setup.yml |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,26 +1,17 @@ | ||
| create schema if not exists v1; | ||
| do | ||
| $$ | ||
| begin | ||
| if not exists ( | ||
| select * | ||
| from pg_catalog.pg_user | ||
| where usename = 'apiuser' | ||
| ) then | ||
| create role my_user password 'SOMEPASSWORD' login; | ||
| end if; | ||
| end | ||
| $$; | ||
| CREATE OR REPLACE FUNCTION v2.translate_array(source CHAR(2), target CHAR(2), q JSON) | ||
| RETURNS TEXT[] AS $BODY$ | ||
| DECLARE | ||
| rez TEXT[]; | ||
| BEGIN | ||
| SELECT | ||
| CASE current_setting('translation_proxy.api.current_engine') | ||
| WHEN 'google' THEN | ||
| translation_proxy.google_translate_array( source, target, q ) | ||
| WHEN 'promt' THEN | ||
| translation_proxy.promt_translate_array( source, target, array_agg( json_array_elements_text(q) ) ) | ||
| END INTO rez; | ||
| RETURN rez; | ||
| END; | ||
| $BODY$ LANGUAGE PLPGSQL SECURITY DEFINER; | ||
|
|
||
| grant usage on schema v1 to apiuser; | ||
|
|
||
| create or replace function v1.google_translate_array(source char(2), target char(2), q json) returns text[] as $$ | ||
| select * from google_translate.translate_array(source, target, q); | ||
| $$ language sql security definer; | ||
|
|
||
| create or replace function v1.google_translate(source char(2), target char(2), q text) returns text as $$ | ||
| select * from google_translate.translate(source, target, q); | ||
| $$ language sql security definer; | ||
|
|
||
| grant execute on function v1.google_translate_array(char, char, json) to apiuser; | ||
| grant execute on function v1.google_translate(char, char, text) to apiuser; | ||
| GRANT EXECUTE ON FUNCTION v2.translate_array(CHAR(2), CHAR(2), JSON) TO apiuser; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| ALTER DATABASE DBNAME SET translation_proxy.api.current_engine = 'CURRENT_API_ENGINE'; | ||
|
|
||
| CREATE SCHEMA IF NOT EXISTS v2; | ||
| DO | ||
| $$ | ||
| BEGIN | ||
| IF NOT EXISTS ( | ||
| SELECT * | ||
| FROM pg_catalog.pg_user | ||
| WHERE usename = 'apiuser' | ||
| ) THEN | ||
| CREATE ROLE apiuser PASSWORD 'APIUSER-PASSWORD' LOGIN; | ||
| END IF; | ||
| END | ||
| $$; | ||
|
|
||
| GRANT USAGE ON SCHEMA v2 TO apiuser; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| -- main function is the bing_translate( source char(2), target char(2), qs text[] ) | ||
|
|
||
| -- api_key | ||
| CREATE OR REPLACE FUNCTION translation_proxy._bing_get_token_curl(text) RETURNS INTEGER AS $$ | ||
| #!/bin/sh | ||
| KEY=$1 | ||
| ACCESS_TOKEN = `curl -X POST -H "content-type: application/x-www-form-urlencoded" \ | ||
| -d "grant_type=client_credentials&client_id=$CLIENTID&client_secret=$CLIENTSECRET&scope=http://api.microsofttranslator.com" \ | ||
| https://datamarket.accesscontrol.windows.net/v2/OAuth2-13 | grep -Po '"access_token":.*?[^\\]",'` | ||
|
|
||
| curl -X POST --header 'Ocp-Apim-Subscription-Key: ${KEY}' --data "" 'https://api.cognitive.microsoft.com/sts/v1.0/issueToken' 2>/dev/null | ||
| $$ LANGUAGE plsh; | ||
|
|
||
| CREATE OR REPLACE FUNCTION translation_proxy._bing_login() RETURNS BOOLEAN AS $$ | ||
| DECLARE | ||
| token TEXT; | ||
| BEGIN | ||
| token := translation_proxy._bing_get_token_curl(current_setting('translation_proxy.bing.api_key')); | ||
| IF token IS NOT NULL AND token <> '' THEN | ||
| UPDATE translation_proxy.authcache SET ( creds, updated ) = ( token, now() ) WHERE api_engine = 'bing'; | ||
| RETURN 't'; | ||
| ELSE | ||
| RETURN 'f'; | ||
| END IF; | ||
| END; | ||
| $$ LANGUAGE plpgsql; | ||
|
|
||
| -- token, source lang, target lang, text, category | ||
| CREATE OR REPLACE FUNCTION translation_proxy._bing_translate_curl(TEXT, CHAR(2), CHAR(2), TEXT) RETURNS TEXT AS $$ | ||
| #!/bin/sh | ||
| TOKEN=$1 | ||
| SRC=$2 | ||
| DST=$3 | ||
| QUERY=$4 | ||
| $CTG=$5 | ||
| curl -X GET -H "Authorization: Bearer $TOKEN" \ | ||
| --data-urlencode "text=$QUERY" \ | ||
| --data-urlencode "from=$SRC" \ | ||
| --data-urlencode "to=$DST" \ | ||
| --data-urlencode "category=$CTG" \ | ||
| 'https://api.cognitive.microsoft.com/sts/v1.0/Translate' 2>/dev/null | ||
| $$ LANGUAGE plsh; | ||
|
|
||
| CREATE OR REPLACE FUNCTION translation_proxy.bing_translate(source CHAR(2), target CHAR(2), qs TEXT[], profile TEXT DEFAULT '') | ||
| RETURNS text[] as $$ | ||
| BEGIN | ||
| END; | ||
| $$ LANGUAGE plpgsql; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| -- Microsoft Bing | ||
| ALTER DATABASE DBNAME SET translation_proxy.bing.api_key = 'YOUR_BING_API_KEY'; | ||
| ALTER DATABASE DBNAME SET translation_proxy.bing.key_expiration = 'BING_TOKEN_EXPIRATION'; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,100 @@ | ||
| CREATE SCHEMA IF NOT EXISTS translation_proxy; | ||
| CREATE EXTENSION IF NOT EXISTS plsh; | ||
| CREATE EXTENSION IF NOT EXISTS plpython2u; | ||
|
|
||
| CREATE TYPE translation_proxy.api_engine_type AS ENUM ('google', 'promt', 'bing'); | ||
|
|
||
| CREATE TABLE translation_proxy.cache( | ||
| id BIGSERIAL PRIMARY KEY, | ||
| source char(2), -- if this is NULL, it need to be detected | ||
| target char(2) NOT NULL, | ||
| q TEXT NOT NULL, | ||
| result TEXT, -- if this is NULL, it need to be translated | ||
| profile TEXT NOT NULL DEFAULT '', | ||
| created TIMESTAMP NOT NULL DEFAULT now(), | ||
| api_engine translation_proxy.api_engine_type NOT NULL, | ||
| encoded TEXT -- urlencoded string for GET request. Is null after an successfull translation. | ||
| ); | ||
|
|
||
| CREATE UNIQUE INDEX u_cache_q_source_target ON translation_proxy.cache | ||
| USING btree(md5(q), source, target, api_engine, profile); | ||
| CREATE INDEX cache_created ON translation_proxy.cache ( created ); | ||
| COMMENT ON TABLE translation_proxy.cache IS 'The cache for API calls of the Translation proxy'; | ||
|
|
||
| -- trigger, that URLencodes query in cache, when no translation is given | ||
| CREATE OR REPLACE FUNCTION translation_proxy._urlencode_fields() | ||
| RETURNS TRIGGER AS $BODY$ | ||
| from urllib import quote_plus | ||
| TD['new']['encoded'] = quote_plus( TD['new']['q'] ) | ||
| return 'MODIFY' | ||
| $BODY$ LANGUAGE plpython2u; | ||
|
|
||
| CREATE TRIGGER _prepare_for_fetch BEFORE INSERT ON translation_proxy.cache | ||
| FOR EACH ROW | ||
| WHEN (NEW.result IS NULL) | ||
| EXECUTE PROCEDURE translation_proxy._urlencode_fields(); | ||
|
|
||
| -- cookies, oauth keys and so on | ||
| CREATE TABLE translation_proxy.authcache( | ||
| api_engine translation_proxy.api_engine_type NOT NULL, | ||
| creds TEXT, | ||
| updated TIMESTAMP NOT NULL DEFAULT now() | ||
| ); | ||
|
|
||
| CREATE UNIQUE INDEX u_authcache_engine ON translation_proxy.authcache ( api_engine ); | ||
|
|
||
| COMMENT ON TABLE translation_proxy.authcache IS 'Translation API cache for remote authorization keys'; | ||
|
|
||
| INSERT INTO translation_proxy.authcache (api_engine) VALUES ('google'), ('promt'), ('bing') | ||
| ON CONFLICT DO NOTHING; | ||
|
|
||
| CREATE OR REPLACE FUNCTION translation_proxy._save_cookie(engine translation_proxy.api_engine_type, cookie TEXT) | ||
| RETURNS VOID AS $$ | ||
| BEGIN | ||
| UPDATE translation_proxy.authcache | ||
| SET ( creds, updated ) = ( cookie, now() ) | ||
| WHERE api_engine = engine; | ||
| END; | ||
| $$ LANGUAGE plpgsql; | ||
|
|
||
| CREATE OR REPLACE FUNCTION translation_proxy._load_cookie(engine translation_proxy.api_engine_type) | ||
| RETURNS TEXT AS $$ | ||
| DECLARE | ||
| cookie TEXT; | ||
| BEGIN | ||
| SELECT creds INTO cookie FROM translation_proxy.authcache | ||
| WHERE api_engine = engine AND | ||
| updated > ( now() - current_setting('translation_proxy.promt.login_timeout')::INTERVAL ) | ||
| AND creds IS NOT NULL AND creds <> '' | ||
| LIMIT 1; | ||
| RETURN cookie; | ||
| END; | ||
| $$ LANGUAGE plpgsql; | ||
|
|
||
| CREATE OR REPLACE FUNCTION translation_proxy._find_detected_language(qs TEXT, engine translation_proxy.api_engine_type) | ||
| RETURNS TEXT AS $$ | ||
| DECLARE | ||
| lng CHAR(2); | ||
| BEGIN | ||
| SELECT lang INTO lng FROM translation_proxy.cache | ||
| WHERE api_engine = engine AND q = qs AND lang IS NOT NULL | ||
| LIMIT 1; | ||
| RETURN lng; | ||
| END; | ||
| $$ LANGUAGE plpgsql; | ||
|
|
||
| -- adding new parameter to url until it exceeds the limit of 2000 bytes | ||
| CREATE OR REPLACE FUNCTION translation_proxy._urladd( url TEXT, a TEXT ) RETURNS TEXT AS $$ | ||
| from urllib import quote_plus | ||
| r = url + quote_plus( a ) | ||
| if len(r) > 1999 : | ||
| plpy.error('URL length is over, time to fetch.', sqlstate = 'EOURL') | ||
| return r | ||
| $$ LANGUAGE plpython2u; | ||
|
|
||
| -- urlencoding utility | ||
| CREATE OR REPLACE FUNCTION translation_proxy._urlencode(q TEXT) | ||
| RETURNS TEXT AS $BODY$ | ||
| from urllib import quote_plus | ||
| return quote_plus( q ) | ||
| $BODY$ LANGUAGE plpython2u; |
Empty file.
Oops, something went wrong.