From a49df268aacce9622adfda914c5dee6a226bf282 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Thu, 10 Apr 2014 13:42:20 +0800 Subject: [PATCH 01/61] add emysql_pool to manager pools --- src/emysql_pool.erl | 176 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 src/emysql_pool.erl diff --git a/src/emysql_pool.erl b/src/emysql_pool.erl new file mode 100644 index 00000000..5d741158 --- /dev/null +++ b/src/emysql_pool.erl @@ -0,0 +1,176 @@ +%%%------------------------------------------------------------------- +%%% @author Jack Tang +%%% @copyright (C) 2014, Jack Tang +%%% @doc +%%% +%%% @end +%%% Created : 10 Apr 2014 by Jack Tang +%%%------------------------------------------------------------------- +-module(emysql_pool). + +-behaviour(gen_server). + +%% API +-export([start_link/1]). + +%% gen_server callbacks +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3]). + +-define(SERVER, ?MODULE). + +-record(state, {pools}). + +%%%=================================================================== +%%% API +%%%=================================================================== + +%%-------------------------------------------------------------------- +%% @doc +%% Starts the server +%% +%% @spec start_link() -> {ok, Pid} | ignore | {error, Error} +%% @end +%%-------------------------------------------------------------------- +pool(Id) -> + gen_server:call(?SERVER, {pool, Id}). + +%%-------------------------------------------------------------------- +%% @doc +%% Starts the server +%% +%% @spec start_link() -> {ok, Pid} | ignore | {error, Error} +%% @end +%%-------------------------------------------------------------------- +start_link(MySqlOpts) -> + gen_server:start_link({local, ?SERVER}, ?MODULE, [MySqlOpts], []). + +%%%=================================================================== +%%% gen_server callbacks +%%%=================================================================== + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Initiates the server +%% +%% @spec init(Args) -> {ok, State} | +%% {ok, State, Timeout} | +%% ignore | +%% {stop, Reason} +%% @end +%%-------------------------------------------------------------------- +init([undefined]) -> + {ok, #state{pools = []}}; +init([MySqlOpt]) -> % MySqlOpt is proplist + Pools = start_pools(MySqlOpt), + {ok, #state{pools = Pools}}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling call messages +%% +%% @spec handle_call(Request, From, State) -> +%% {reply, Reply, State} | +%% {reply, Reply, State, Timeout} | +%% {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, Reply, State} | +%% {stop, Reason, State} +%% @end +%%-------------------------------------------------------------------- +handle_call({pool, Id}, _From, #state{pools = Pools}=State) -> + Reply = case lists:member(Id, Pools) of + true -> {ok, ok}; + false -> {error, not_found} + end, + {reply, Reply, State}; +handle_call(_Request, _From, State) -> + Reply = ok, + {reply, Reply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling cast messages +%% +%% @spec handle_cast(Msg, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} +%% @end +%%-------------------------------------------------------------------- +handle_cast(_Msg, State) -> + {noreply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling all non call/cast messages +%% +%% @spec handle_info(Info, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} +%% @end +%%-------------------------------------------------------------------- +handle_info(_Info, State) -> + {noreply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% This function is called by a gen_server when it is about to +%% terminate. It should be the opposite of Module:init/1 and do any +%% necessary cleaning up. When it returns, the gen_server terminates +%% with Reason. The return value is ignored. +%% +%% @spec terminate(Reason, State) -> void() +%% @end +%%-------------------------------------------------------------------- +terminate(_Reason, #state{pools = Pools} = State) -> + lists:foreach( + fun(Pool) -> + emysql:remove_pool(Pool) + end, Pools), + ok. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Convert process state when code is changed +%% +%% @spec code_change(OldVsn, State, Extra) -> {ok, NewState} +%% @end +%%-------------------------------------------------------------------- +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== + +start_pools(MySqlOpt) -> + Host = proplists:get_value(host, MySqlOpt, "localhost"), + Port = proplists:get_value(port, MySqlOpt, 3306), + UserName = proplists:get_value(username, MySqlOpt, "root"), + Password = proplists:get_value(password, MySqlOpt), + Encoding = proplists:get_value(encoding, MySqlOpt, utf8), + Database = proplists:get_value(database, MySqlOpt), + Pools = proplists:get_value(pools, MySqlOpt, []), + + lists:foldl( + fun({PoolName, PoolSize}, AccIn) -> + case emysql:add_pool(PoolName, PoolSize, UserName, Password, + Host, Port, Database, Encoding) of + ok -> [PoolName | AccIn]; + {error, Reason} -> + io:format("EMysqlPool add pool<~p> failed: ~p", + [PoolName, Reason]), + {error, Reason} + end + end, [], Pools). + From 2414e7c31fbcbc066dbd453d60669c17de0464c4 Mon Sep 17 00:00:00 2001 From: slepher Date: Thu, 10 Apr 2014 16:11:11 +0800 Subject: [PATCH 02/61] add transaction --- src/emysql.erl | 12 ++++++++++++ src/emysql_conn.erl | 31 +++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/src/emysql.erl b/src/emysql.erl index a373de9a..17f889ba 100644 --- a/src/emysql.erl +++ b/src/emysql.erl @@ -114,6 +114,8 @@ default_timeout/0 ]). +-export([transaction/2, transaction/3, abort/1]). + %% Result Conversion API -export([ as_dict/1, @@ -631,6 +633,16 @@ execute(PoolId, StmtName, Args, Timeout, nonblocking) when is_atom(StmtName), is unavailable end. +transaction(PoolId, Fun) -> + transaction(PoolId, Fun, default_timeout()). + +transaction(PoolId, Fun, Timeout) -> + Connection = emysql_conn_mgr:wait_for_connection(PoolId), + monitor_work(Connection, Timeout, [Connection, transaction, Fun]). + +abort(Reason) -> + throw(Reason). + %% @doc Return the field names of a result packet %% @end -spec field_names(Result) -> [Name] diff --git a/src/emysql_conn.erl b/src/emysql_conn.erl index 01a5c74b..983e9d78 100644 --- a/src/emysql_conn.erl +++ b/src/emysql_conn.erl @@ -86,6 +86,28 @@ set_encoding(Connection, Encoding) -> canonicalize_query(Q) when is_binary(Q) -> Q; canonicalize_query(QL) when is_list(QL) -> iolist_to_binary(QL). +execute(Connection, transaction, Fun) when is_function(Fun) -> + case begin_transaction(Connection) of + #ok_packet{} -> + try Fun(Connection) of + Val -> + case commit_transaction(Connection) of + #ok_packet{} -> + {atomic, Val}; + #error_packet{} = ErrorPacket -> + {aborted, {commit_error, ErrorPacket}} + end + catch + throw:Reason -> + rollback_transaction(Connection), + {aborted, Reason}; + Class:Exception -> + rollback_transaction(Connection), + erlang:raise(Class, Exception, erlang:get_stacktrace()) + end; + #error_packet{} = ErrorPacket -> + {aborted, {begin_error, ErrorPacket}} + end; execute(Connection, StmtName, []) when is_atom(StmtName) -> prepare_statement(Connection, StmtName), StmtNameBin = atom_to_binary(StmtName, utf8), @@ -140,6 +162,15 @@ unprepare(Connection, Name) -> Packet = <>, % todo: utf8? send_recv(Connection, Packet). +begin_transaction(Connection) -> + emysql_conn:execute(Connection, <<"BEGIN">>, []). + +rollback_transaction(Connection) -> + emysql_conn:execute(Connection, <<"ROLLBACK">>, []). + +commit_transaction(Connection) -> + emysql_conn:execute(Connection, <<"COMMIT">>, []). + open_n_connections(PoolId, N) -> case emysql_conn_mgr:find_pool(PoolId, emysql_conn_mgr:pools()) of {Pool, _} -> From acd2a2df1bba6942eec99da94921daf3061f27ba Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Thu, 24 Apr 2014 15:20:23 +0800 Subject: [PATCH 03/61] add ActiveRecord style query interface --- include/emysql.hrl | 3 ++ rebar.config | 8 ++++ src/emysql_query.erl | 105 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 116 insertions(+) create mode 100644 src/emysql_query.erl diff --git a/include/emysql.hrl b/include/emysql.hrl index 1320289f..22f08d3c 100644 --- a/include/emysql.hrl +++ b/include/emysql.hrl @@ -180,3 +180,6 @@ % number of result set columns. -define(SERVER_STATUS_METADATA_CHANGED, 1024). +% Wrap the query result in erlang record +-define(AS_REC(Rec), [Rec, record_info(fields, Rec)]). + diff --git a/rebar.config b/rebar.config index 9324cb18..6a978201 100644 --- a/rebar.config +++ b/rebar.config @@ -4,3 +4,11 @@ {"linux|bsd|darwin|solaris", compile, "escript ./support/crypto_compat.escript"}, {"win32", compile, "escript.exe support/crypto_compat.escript"} ]}. + +{deps, [ + {'espec', ".*", { git, "git://github.com/lucaspiller/espec.git", "master" } }, + {'erl_utils', ".*", { git, "git://github.com/jacktang/erl_utils.git", "master"} } + ]}. + + +{xref_warnings, true}. diff --git a/src/emysql_query.erl b/src/emysql_query.erl new file mode 100644 index 00000000..6e52ab12 --- /dev/null +++ b/src/emysql_query.erl @@ -0,0 +1,105 @@ +%%%------------------------------------------------------------------- +%%% @author Jack Tang +%%% @copyright (C) 2014, Jack Tang +%%% @doc +%%% +%%% @end +%%% Created : 24 Apr 2014 by Jack Tang +%%%------------------------------------------------------------------- +-module(emysql_query). + +%% API +-export([]). + +%%%=================================================================== +%%% API +%%%=================================================================== + +%%-------------------------------------------------------------------- +%% @doc +%% e.g: find(my_pool, users, [ {select, [name, age]}, +%% {where, ["age > ?", Age]}, +%% {order, "id desc"} ], ?AS_REC(user)) +%% +%% @spec +%% @end +%%-------------------------------------------------------------------- +find_first(ConnOrPool, Table, SqlOptions, AsRec) -> + NSqlOptions = proplists:delete(limit, SqlOptions). + find(ConnOrPool, Table, [{limit, 1} | NSqlOptions], AsRec). + + +%%-------------------------------------------------------------------- +%% @doc +%% e.g: find(my_pool, users, [ {select, [name, age]}, +%% {where, ["age > ?", Age]}, +%% {order, "id desc"}, +%% {limit, 100 }], ?AS_REC(user)) +%% +%% @spec +%% @end +%%-------------------------------------------------------------------- +find(ConnOrPool, Table, SqlOptions, [Rec, Fileds] = _AsRec) -> + {FindSql, CondVals} = build_sql(Table, SqlOptions), + Result = case ConnOrPool of + #emysql_connection{} = Conn -> + emysql_conn:execute(Conn, FindSql, CondVals); + Pool -> + emysql:execute(Pool, FindSql, CondVals) + end, + emysql_util:as_record(Result, Rec, RecFields). + +%%-------------------------------------------------------------------- +%% @doc +%% e.g: find(my_pool, ["SELECT name FROM users where age > ? and time > ?", 12, "2013-12-03" ]) +%% +%% +%% @spec +%% @end +%%-------------------------------------------------------------------- +find(ConnPool, [RawSql | Values]) -> + StmtName = md5_hex(RawSql), + emysql:prepare(StmtName, type_utils:any_to_binay(RawSql)), + case emysql:execute(ConnPool, StmtName, Values) of + #ok_packet{} = OkPacket -> OkPacket; + #error_packet{} = ErrorPacket -> ErrorPacket + end. + +%%-------------------------------------------------------------------- +%% @doc +%% +%% e.g: +%% Fun = fun(#user{} = U) -> +%% do_something_to(U) +%% end, +%% find(my_pool, users, [ {select, [name, age]}, +%% {where, ["age > ?", Age]}, +%% {order, "id desc"}, +%% {limit, 100 }], 10, ?AS_REC(user), Fun) +%% +%% @spec +%% @end +%%-------------------------------------------------------------------- +find_each(Connection, Table, SqlOptions, Size, AsRec, Fun) -> + ok. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== +build_sql(Table, SqlOptions) -> + SelectF = proplists:get_value(select, SqlOptions, "*") + + Select = "SELECT " + SelectF + " FROM " ++ type_utils:any_to_list(Table), + {Where, CondVals} = case proplists:get_value(where, SqlOptions) of + undefined -> " "; + [Cond | Values] -> {Cond, Values} + end, + FindSql = lists:append(Select, Where), + {FindSql, CondVals}. + + +md5_hex(S) -> + lists:flatten([io_lib:format("~.16b",[N]) || <> <= erlang:md5(S)]). + + + From 72cb0dfc9b2f22fee2428e0b71cdfeaeaa3ee893 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Thu, 24 Apr 2014 15:22:00 +0800 Subject: [PATCH 04/61] format code --- src/emysql_pool.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/emysql_pool.erl b/src/emysql_pool.erl index 5d741158..4377f425 100644 --- a/src/emysql_pool.erl +++ b/src/emysql_pool.erl @@ -164,7 +164,10 @@ start_pools(MySqlOpt) -> lists:foldl( fun({PoolName, PoolSize}, AccIn) -> - case emysql:add_pool(PoolName, PoolSize, UserName, Password, + case emysql:add_pool(PoolName, + PoolSize, + UserName, + Password, Host, Port, Database, Encoding) of ok -> [PoolName | AccIn]; {error, Reason} -> From 25c20045335289387c3423aecf9df66988e48c9d Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Mon, 28 Apr 2014 11:18:14 +0800 Subject: [PATCH 05/61] add deps to gitignore --- .gitignore | 1 + src/emysql_query.erl | 14 +++++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 87e2ec70..4858b743 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ test/ct_log_cache logs/* .#* include/crypto_compat.hrl +deps/* diff --git a/src/emysql_query.erl b/src/emysql_query.erl index 6e52ab12..f6176b95 100644 --- a/src/emysql_query.erl +++ b/src/emysql_query.erl @@ -11,6 +11,9 @@ %% API -export([]). + +-include("emysql.hrl"). + %%%=================================================================== %%% API %%%=================================================================== @@ -25,7 +28,7 @@ %% @end %%-------------------------------------------------------------------- find_first(ConnOrPool, Table, SqlOptions, AsRec) -> - NSqlOptions = proplists:delete(limit, SqlOptions). + NSqlOptions = proplists:delete(limit, SqlOptions), find(ConnOrPool, Table, [{limit, 1} | NSqlOptions], AsRec). @@ -39,7 +42,7 @@ find_first(ConnOrPool, Table, SqlOptions, AsRec) -> %% @spec %% @end %%-------------------------------------------------------------------- -find(ConnOrPool, Table, SqlOptions, [Rec, Fileds] = _AsRec) -> +find(ConnOrPool, Table, SqlOptions, [Rec, RecFields] = _AsRec) -> {FindSql, CondVals} = build_sql(Table, SqlOptions), Result = case ConnOrPool of #emysql_connection{} = Conn -> @@ -87,9 +90,9 @@ find_each(Connection, Table, SqlOptions, Size, AsRec, Fun) -> %%% Internal functions %%%=================================================================== build_sql(Table, SqlOptions) -> - SelectF = proplists:get_value(select, SqlOptions, "*") + SelectF = proplists:get_value(select, SqlOptions, "*"), - Select = "SELECT " + SelectF + " FROM " ++ type_utils:any_to_list(Table), + Select = "SELECT " ++ SelectF ++ " FROM " ++ type_utils:any_to_list(Table), {Where, CondVals} = case proplists:get_value(where, SqlOptions) of undefined -> " "; [Cond | Values] -> {Cond, Values} @@ -99,7 +102,8 @@ build_sql(Table, SqlOptions) -> md5_hex(S) -> - lists:flatten([io_lib:format("~.16b",[N]) || <> <= erlang:md5(S)]). + S. + % lists:flatten([io_lib:format("~.16b",[N]) || <<>> <= erlang:md5(S)]). From 5bd077318f6ae466ca8b0bdd1f2497d5905841cd Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Mon, 28 Apr 2014 13:01:17 +0800 Subject: [PATCH 06/61] adjust to emysql:add_pool/2 --- src/emysql_pool.erl | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/emysql_pool.erl b/src/emysql_pool.erl index 4377f425..52cb07d8 100644 --- a/src/emysql_pool.erl +++ b/src/emysql_pool.erl @@ -66,7 +66,7 @@ start_link(MySqlOpts) -> %%-------------------------------------------------------------------- init([undefined]) -> {ok, #state{pools = []}}; -init([MySqlOpt]) -> % MySqlOpt is proplist +init([MySqlOpt]) when is_list(MySqlOpt) -> % MySqlOpt is proplist Pools = start_pools(MySqlOpt), {ok, #state{pools = Pools}}. @@ -156,7 +156,7 @@ code_change(_OldVsn, State, _Extra) -> start_pools(MySqlOpt) -> Host = proplists:get_value(host, MySqlOpt, "localhost"), Port = proplists:get_value(port, MySqlOpt, 3306), - UserName = proplists:get_value(username, MySqlOpt, "root"), + UserName = proplists:get_value(user, MySqlOpt, "root"), Password = proplists:get_value(password, MySqlOpt), Encoding = proplists:get_value(encoding, MySqlOpt, utf8), Database = proplists:get_value(database, MySqlOpt), @@ -165,10 +165,13 @@ start_pools(MySqlOpt) -> lists:foldl( fun({PoolName, PoolSize}, AccIn) -> case emysql:add_pool(PoolName, - PoolSize, - UserName, - Password, - Host, Port, Database, Encoding) of + [{size, type_utils:any_to_integer(PoolSize)}, + {user, type_utils:any_to_list(UserName)}, + {password, type_utils:any_to_list(Password)}, + {host, type_utils:any_to_list(Host)}, + {port, type_utils:any_to_integer(Port)}, + {database, type_utils:any_to_list(Database)}, + {encoding, Encoding}] ) of ok -> [PoolName | AccIn]; {error, Reason} -> io:format("EMysqlPool add pool<~p> failed: ~p", From 28a08fc8925eaecacaaa59b7462948fd892b2759 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Tue, 29 Apr 2014 21:35:01 +0800 Subject: [PATCH 07/61] add test database config --- src/emysql.app.src | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/emysql.app.src b/src/emysql.app.src index 1eca61a5..230c9c24 100644 --- a/src/emysql.app.src +++ b/src/emysql.app.src @@ -10,6 +10,11 @@ {registered, [emysql_conn_mgr, emysql_sup]}, {applications, [kernel, stdlib, crypto]}, {env, [ - {default_timeout, 5000}, - {conn_test_period, 28000}]} + {default_timeout, 5000}, + {conn_test_period, 28000}, + {default_conn_opt, [ {host, "localhost"}, + {user, "root"}, + {database, test} + ]} + ]} ]}. From a14465fc67226b5b15dee7e1489926a5a56ded37 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Tue, 29 Apr 2014 21:35:43 +0800 Subject: [PATCH 08/61] complete find and find_first --- src/emysql_query.erl | 119 ++++++++++++++++++++++++++++++------------- 1 file changed, 85 insertions(+), 34 deletions(-) diff --git a/src/emysql_query.erl b/src/emysql_query.erl index f6176b95..b9615dc4 100644 --- a/src/emysql_query.erl +++ b/src/emysql_query.erl @@ -9,7 +9,8 @@ -module(emysql_query). %% API --export([]). +-export([find/2, find/4]). +-export([find_first/4, find_each/6]). -include("emysql.hrl"). @@ -18,19 +19,22 @@ %%% API %%%=================================================================== + %%-------------------------------------------------------------------- %% @doc -%% e.g: find(my_pool, users, [ {select, [name, age]}, -%% {where, ["age > ?", Age]}, -%% {order, "id desc"} ], ?AS_REC(user)) +%% e.g: find(my_pool, ["SELECT name FROM users where age > ? and time > ?", 12, "2013-12-03" ]) +%% %% %% @spec %% @end %%-------------------------------------------------------------------- -find_first(ConnOrPool, Table, SqlOptions, AsRec) -> - NSqlOptions = proplists:delete(limit, SqlOptions), - find(ConnOrPool, Table, [{limit, 1} | NSqlOptions], AsRec). - +find(ConnPool, [RawSql | Values]) -> + StmtName = type_utils:md5_hex(RawSql), + emysql:prepare(StmtName, type_utils:any_to_binay(RawSql)), + case emysql:execute(ConnPool, StmtName, Values) of + #ok_packet{} = OkPacket -> OkPacket; + #error_packet{} = ErrorPacket -> ErrorPacket + end. %%-------------------------------------------------------------------- %% @doc @@ -43,30 +47,29 @@ find_first(ConnOrPool, Table, SqlOptions, AsRec) -> %% @end %%-------------------------------------------------------------------- find(ConnOrPool, Table, SqlOptions, [Rec, RecFields] = _AsRec) -> - {FindSql, CondVals} = build_sql(Table, SqlOptions), + {Sql, CondVals} = build_sql(Table, SqlOptions), Result = case ConnOrPool of #emysql_connection{} = Conn -> - emysql_conn:execute(Conn, FindSql, CondVals); + emysql_conn:execute(Conn, Sql, CondVals); Pool -> - emysql:execute(Pool, FindSql, CondVals) + emysql:execute(Pool, Sql, CondVals) end, emysql_util:as_record(Result, Rec, RecFields). + %%-------------------------------------------------------------------- %% @doc -%% e.g: find(my_pool, ["SELECT name FROM users where age > ? and time > ?", 12, "2013-12-03" ]) -%% +%% e.g: find(my_pool, users, [ {select, [name, age]}, +%% {where, ["age > ?", Age]}, +%% {order, "id desc"} ], ?AS_REC(user)) %% %% @spec %% @end %%-------------------------------------------------------------------- -find(ConnPool, [RawSql | Values]) -> - StmtName = md5_hex(RawSql), - emysql:prepare(StmtName, type_utils:any_to_binay(RawSql)), - case emysql:execute(ConnPool, StmtName, Values) of - #ok_packet{} = OkPacket -> OkPacket; - #error_packet{} = ErrorPacket -> ErrorPacket - end. +find_first(ConnOrPool, Table, SqlOptions, AsRec) -> + NSqlOptions = proplists:delete(limit, SqlOptions), + find(ConnOrPool, Table, [{limit, 1} | NSqlOptions], AsRec). + %%-------------------------------------------------------------------- %% @doc @@ -75,35 +78,83 @@ find(ConnPool, [RawSql | Values]) -> %% Fun = fun(#user{} = U) -> %% do_something_to(U) %% end, -%% find(my_pool, users, [ {select, [name, age]}, +%% find(my_pool, users, [ {select, ["name", "age"]}, %% {where, ["age > ?", Age]}, %% {order, "id desc"}, -%% {limit, 100 }], 10, ?AS_REC(user), Fun) +%% {limit, 100 } ], 10, ?AS_REC(user), Fun) %% %% @spec %% @end %%-------------------------------------------------------------------- -find_each(Connection, Table, SqlOptions, Size, AsRec, Fun) -> - ok. +find_each(ConnOrPool, Table, SqlOptions, BatchSize, AsRec, Fun) -> + {FindSql, CountSql, CondVals} = build_sql(Table, SqlOptions), + Result = case ConnOrPool of + #emysql_connection{} = Conn -> + emysql_conn:execute(Conn, CountSql, CondVals); + Pool -> + emysql:execute(Pool, CountSql, CondVals) + end, + case Result of + #ok_packet{} -> + Total = 1000, + Limit = proplists:get_value(limit, SqlOptions, 0), + Remain = 10, % (Total > Limit ? Limit : Total), + do_find_each(ConnOrPool, Table, FindSql, CondVals, BatchSize, AsRec, Fun, Remain); + #error_packet{} -> + ok + end. + +do_find_each(ConnOrPool, Table, Sql, CondVals, BatchSize, [Rec, RecFields] = AsRec, Fun, Remain) -> + Result = case ConnOrPool of + #emysql_connection{} = Conn -> + emysql_conn:execute(Conn, Sql, CondVals); + Pool -> + emysql:execute(Pool, Sql, CondVals) + end, + lists:foreach( + fun(Item) -> + Fun(emysql_util:as_record(Result, Rec, RecFields)) + end, Result), + + case Remain - BatchSize > 0 of + true -> + do_find_each(ConnOrPool, Table, Sql, CondVals, BatchSize, AsRec, Fun, (Remain - BatchSize)); + false -> + ok + end. + %%%=================================================================== %%% Internal functions %%%=================================================================== build_sql(Table, SqlOptions) -> - SelectF = proplists:get_value(select, SqlOptions, "*"), + SelectFields = + case proplists:get_value(select, SqlOptions) of + undefined -> "*"; + V when is_list(V) -> string:join(V, ","); + _ -> throw(bad_sql_select) + end, - Select = "SELECT " ++ SelectF ++ " FROM " ++ type_utils:any_to_list(Table), {Where, CondVals} = case proplists:get_value(where, SqlOptions) of - undefined -> " "; - [Cond | Values] -> {Cond, Values} + undefined -> ""; + [Cond | Values] -> {"WHERE " ++ Cond, Values} end, - FindSql = lists:append(Select, Where), - {FindSql, CondVals}. - + + Order = case proplists:get_value(order, SqlOptions) of + undefined -> ""; + OrderBy -> " ORDER BY " ++ type_utils:any_to_list(OrderBy) + end, + Limit = case proplists:get_value(limit, SqlOptions) of + undefined -> ""; + [LV1, LV2] -> + " LIMIT " ++ type_utils:any_to_list(LV1) ++ ", " ++ type_utils:any_to_list(LV2); + LV3 -> " LIMIT " ++ type_utils:any_to_list(LV3) + end, -md5_hex(S) -> - S. - % lists:flatten([io_lib:format("~.16b",[N]) || <<>> <= erlang:md5(S)]). + Table2 = type_utils:any_to_list(Table), + FindSql = string:join(["SELECT", SelectFields, "FROM", Table2, Where, Order, Limit], " "), + CountSql = string:join(["SELECT COUNT(1) FROM", Table2, Where], " "), + {FindSql, CountSql, CondVals}. From 8f3c2f5cd002cef448b22a4e9bdaacf116c78b97 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Tue, 29 Apr 2014 21:50:13 +0800 Subject: [PATCH 09/61] fix typo --- src/emysql_query.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emysql_query.erl b/src/emysql_query.erl index b9615dc4..84e677c8 100644 --- a/src/emysql_query.erl +++ b/src/emysql_query.erl @@ -30,7 +30,7 @@ %%-------------------------------------------------------------------- find(ConnPool, [RawSql | Values]) -> StmtName = type_utils:md5_hex(RawSql), - emysql:prepare(StmtName, type_utils:any_to_binay(RawSql)), + emysql:prepare(StmtName, type_utils:any_to_binary(RawSql)), case emysql:execute(ConnPool, StmtName, Values) of #ok_packet{} = OkPacket -> OkPacket; #error_packet{} = ErrorPacket -> ErrorPacket From 3cf0a8c7985e4e0930dbafafd8c5863d7c7ebca0 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Tue, 29 Apr 2014 22:12:46 +0800 Subject: [PATCH 10/61] fix emysql:prepare error --- src/emysql_query.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emysql_query.erl b/src/emysql_query.erl index 84e677c8..d4e115c2 100644 --- a/src/emysql_query.erl +++ b/src/emysql_query.erl @@ -29,7 +29,7 @@ %% @end %%-------------------------------------------------------------------- find(ConnPool, [RawSql | Values]) -> - StmtName = type_utils:md5_hex(RawSql), + StmtName = type_utils:any_to_atom("stmt_" ++ type_utils:md5_hex(RawSql)), emysql:prepare(StmtName, type_utils:any_to_binary(RawSql)), case emysql:execute(ConnPool, StmtName, Values) of #ok_packet{} = OkPacket -> OkPacket; From 5342929ab77152ee96884723d807776b14d89e89 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Tue, 29 Apr 2014 22:21:31 +0800 Subject: [PATCH 11/61] refine find --- src/emysql_query.erl | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/emysql_query.erl b/src/emysql_query.erl index d4e115c2..800d6304 100644 --- a/src/emysql_query.erl +++ b/src/emysql_query.erl @@ -31,10 +31,7 @@ find(ConnPool, [RawSql | Values]) -> StmtName = type_utils:any_to_atom("stmt_" ++ type_utils:md5_hex(RawSql)), emysql:prepare(StmtName, type_utils:any_to_binary(RawSql)), - case emysql:execute(ConnPool, StmtName, Values) of - #ok_packet{} = OkPacket -> OkPacket; - #error_packet{} = ErrorPacket -> ErrorPacket - end. + emysql:execute(ConnPool, StmtName, Values). %%-------------------------------------------------------------------- %% @doc From dba13459717376ba213ed8615829ae76e0e8a359 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Wed, 30 Apr 2014 01:13:08 +0800 Subject: [PATCH 12/61] find and find_first works --- src/emysql_query.erl | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/emysql_query.erl b/src/emysql_query.erl index 800d6304..507ad00c 100644 --- a/src/emysql_query.erl +++ b/src/emysql_query.erl @@ -28,10 +28,13 @@ %% @spec %% @end %%-------------------------------------------------------------------- -find(ConnPool, [RawSql | Values]) -> - StmtName = type_utils:any_to_atom("stmt_" ++ type_utils:md5_hex(RawSql)), - emysql:prepare(StmtName, type_utils:any_to_binary(RawSql)), - emysql:execute(ConnPool, StmtName, Values). +find(ConnOrPool, [RawSql | Values]) -> + case ConnOrPool of + #emysql_connection{} = Conn -> + mysql_conn:execute(Conn, RawSql, Values); + Pool -> + emysql:execute(ConnPool, RawSql, Values) + end. %%-------------------------------------------------------------------- %% @doc @@ -44,12 +47,12 @@ find(ConnPool, [RawSql | Values]) -> %% @end %%-------------------------------------------------------------------- find(ConnOrPool, Table, SqlOptions, [Rec, RecFields] = _AsRec) -> - {Sql, CondVals} = build_sql(Table, SqlOptions), + {FindSql, _, CondVals} = build_sql(Table, SqlOptions), Result = case ConnOrPool of #emysql_connection{} = Conn -> - emysql_conn:execute(Conn, Sql, CondVals); + emysql_conn:execute(Conn, FindSql, CondVals); Pool -> - emysql:execute(Pool, Sql, CondVals) + emysql:execute(Pool, FindSql, CondVals) end, emysql_util:as_record(Result, Rec, RecFields). @@ -83,6 +86,9 @@ find_first(ConnOrPool, Table, SqlOptions, AsRec) -> %% @spec %% @end %%-------------------------------------------------------------------- +find_each(ConnOrPool, Table, SqlOptions, AsRec, Fun) -> + find_each(ConnOrPool, Table, SqlOptions, 1000, AsRec, Fun). + find_each(ConnOrPool, Table, SqlOptions, BatchSize, AsRec, Fun) -> {FindSql, CountSql, CondVals} = build_sql(Table, SqlOptions), Result = case ConnOrPool of @@ -133,7 +139,9 @@ build_sql(Table, SqlOptions) -> end, {Where, CondVals} = case proplists:get_value(where, SqlOptions) of - undefined -> ""; + undefined -> {"", [ ]}; + [ ] -> {"", [ ]}; + [ Cond ] -> {"WHERE " ++ Cond, [ ]}; [Cond | Values] -> {"WHERE " ++ Cond, Values} end, From ac8abbf23a2f4f12d3083d6ab6b3830609a8b278 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Wed, 30 Apr 2014 01:13:42 +0800 Subject: [PATCH 13/61] trace sql execution in some cases --- src/emysql_conn.erl | 5 + src/execute_trace.erl | 249 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 254 insertions(+) create mode 100644 src/execute_trace.erl diff --git a/src/emysql_conn.erl b/src/emysql_conn.erl index 983e9d78..3175d567 100644 --- a/src/emysql_conn.erl +++ b/src/emysql_conn.erl @@ -111,10 +111,12 @@ execute(Connection, transaction, Fun) when is_function(Fun) -> execute(Connection, StmtName, []) when is_atom(StmtName) -> prepare_statement(Connection, StmtName), StmtNameBin = atom_to_binary(StmtName, utf8), + execute_trace:execute(StmtName), Packet = <>, send_recv(Connection, Packet); execute(Connection, Query, []) -> QB = canonicalize_query(Query), + execute_trace:execute(raw_query, QB), Packet = <>, send_recv(Connection, Packet); execute(Connection, Query, Args) when (is_list(Query) orelse is_binary(Query)) andalso is_list(Args) -> @@ -125,6 +127,7 @@ execute(Connection, Query, Args) when (is_list(Query) orelse is_binary(Query)) a OK when is_record(OK, ok_packet) -> ParamNamesBin = list_to_binary(string:join([[$@ | integer_to_list(I)] || I <- lists:seq(1, length(Args))], ", ")), % todo: utf8? Packet = <>, % todo: utf8? + execute_trace:execute(StmtName, ParamNamesBin), send_recv(Connection, Packet); Error -> Error @@ -148,6 +151,7 @@ prepare(Connection, Name, Statement) when is_atom(Name) -> prepare(Connection, atom_to_list(Name), Statement); prepare(Connection, Name, Statement) -> StatementBin = encode(Statement, binary), + execute_trace:prepare(Name, Statement), Packet = <>, % todo: utf8? case send_recv(Connection, Packet) of OK when is_record(OK, ok_packet) -> @@ -159,6 +163,7 @@ prepare(Connection, Name, Statement) -> unprepare(Connection, Name) when is_atom(Name)-> unprepare(Connection, atom_to_list(Name)); unprepare(Connection, Name) -> + execute_trace:unprepare(Name), Packet = <>, % todo: utf8? send_recv(Connection, Packet). diff --git a/src/execute_trace.erl b/src/execute_trace.erl new file mode 100644 index 00000000..cb2c4539 --- /dev/null +++ b/src/execute_trace.erl @@ -0,0 +1,249 @@ +%%%------------------------------------------------------------------- +%%% @author Jack Tang +%%% @copyright (C) 2014, Jack Tang +%%% @doc +%%% +%%% @end +%%% Created : 29 Apr 2014 by Jack Tang +%%%------------------------------------------------------------------- +-module(execute_trace). + +-behaviour(gen_server). + +%% API +-export([prepare/2, + unprepare/1, + execute/2, + begin_transaction/0, + commit_transaction/0, + rollback_transaction/0]). + +-export([start_link/0]). + +%% gen_server callbacks +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3]). + +-define(SERVER, ?MODULE). + +-record(state, { stmts }). + +%%%=================================================================== +%%% API +%%%=================================================================== + +%%-------------------------------------------------------------------- +%% @doc +%% +%% @spec +%% @end +%%-------------------------------------------------------------------- +prepare(Name, Statement) -> + gen_server:cast(?SERVER, {prepare, type_utils:any_to_binary(Name), Statement}). + +%%-------------------------------------------------------------------- +%% @doc +%% +%% @spec +%% @end +%%-------------------------------------------------------------------- +unprepare(Name) -> + gen_server:cast(?SERVER, {unprepare, type_utils:any_to_binary(Name)}). + +%%-------------------------------------------------------------------- +%% @doc +%% +%% @spec +%% @end +%%-------------------------------------------------------------------- +execute(StmtName) -> + execute(StmtName, []). +execute(raw_query, Query) -> + gen_server:cast(?SERVER, {raw_query, Query}). +execute(StmtName, Params) -> + gen_server:cast(?SERVER, {execute, type_utils:any_to_binary(StmtName), Statement}). + +%%-------------------------------------------------------------------- +%% @doc +%% +%% @spec +%% @end +%%-------------------------------------------------------------------- +begin_transaction() -> + gen_server:cast(?SERVER, begin_transaction). + +%%-------------------------------------------------------------------- +%% @doc +%% +%% @spec +%% @end +%%-------------------------------------------------------------------- +commit_transaction() -> + gen_server:cast(?SERVER, commit). + + +%%-------------------------------------------------------------------- +%% @doc +%% +%% @spec +%% @end +%%-------------------------------------------------------------------- +rollback_transaction() -> + gen_server:cast(?SERVER, rollback). + + + + +%%-------------------------------------------------------------------- +%% @doc +%% +%% @spec +%% @end +%%-------------------------------------------------------------------- +execute(StmtName, Statement) -> + gen_server:cast(?SERVER, {execute, StmtName, Statement}). + + + +%%-------------------------------------------------------------------- +%% @doc +%% Starts the server +%% +%% @spec start_link() -> {ok, Pid} | ignore | {error, Error} +%% @end +%%-------------------------------------------------------------------- +start_link() -> + gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). + +%%%=================================================================== +%%% gen_server callbacks +%%%=================================================================== + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Initiates the server +%% +%% @spec init(Args) -> {ok, State} | +%% {ok, State, Timeout} | +%% ignore | +%% {stop, Reason} +%% @end +%%-------------------------------------------------------------------- +init([]) -> + {ok, #state{ stmts = orddict:new() }}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling call messages +%% +%% @spec handle_call(Request, From, State) -> +%% {reply, Reply, State} | +%% {reply, Reply, State, Timeout} | +%% {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, Reply, State} | +%% {stop, Reason, State} +%% @end +%%-------------------------------------------------------------------- +handle_call(_Request, _From, State) -> + Reply = ok, + {reply, Reply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling cast messages +%% +%% @spec handle_cast(Msg, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} +%% @end +%%-------------------------------------------------------------------- +handle_cast({prepare, Name, Statement}, #state{stmts = Stmts}=State) -> + NStmts = case orddict:find(Name, Stmts) of + error -> + orddict:store(Name, [Statement, 1], Stmts); + {ok, [Statement, Ref]} -> + orddict:store(Name, [Statement, Ref + 1], Stmts); + {ok, [TheStatemnt, _]} -> + lager:error("Prepare statement (~p) conflicted: ~p <-> ~p", + [Name, TheStatement, Statement]), + Stmts + end, + {noreply, State#state{stmts = NStmts}}; + +handle_cast({unprepare, Name}, #state{stmt = Stmts} = State) -> + NStmts = case orddict:find(Name, Stmts) of + error -> Stmts; + {ok, [Statement, 1]} -> + orddict:erase(Name, Stmts); + {ok, [Statement, Ref]} -> + orddict:store(Name, [Statement, Ref - 1], Stmts) + end, + {noreply, State#state{stmts = NStmts}}; + +handle_cast({execute, Name, Params}, #state{stmts = Stmts}=State) -> + case orddict:find(Name, Stmts) of + error -> + lager:warning("No statement (~p) found", [Name]); + {ok, Statement} -> + lager:debug("Execute ~p: ~p~nParams: ~p", + [Name, Statment, Params]) + end, + {noreply, State}; + +handle_cast({raw_query, Query}, #state{}=State) -> + lager:debug("Query: ~p", [Query]), + {noreply, State}; + + +handle_cast(_Msg, State) -> + {noreply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling all non call/cast messages +%% +%% @spec handle_info(Info, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} +%% @end +%%-------------------------------------------------------------------- +handle_info(_Info, State) -> + {noreply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% This function is called by a gen_server when it is about to +%% terminate. It should be the opposite of Module:init/1 and do any +%% necessary cleaning up. When it returns, the gen_server terminates +%% with Reason. The return value is ignored. +%% +%% @spec terminate(Reason, State) -> void() +%% @end +%%-------------------------------------------------------------------- +terminate(_Reason, _State) -> + ok. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Convert process state when code is changed +%% +%% @spec code_change(OldVsn, State, Extra) -> {ok, NewState} +%% @end +%%-------------------------------------------------------------------- +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== From 3b107c99845a3cb91140c7c4378a5f6827c96d2b Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Wed, 30 Apr 2014 01:16:37 +0800 Subject: [PATCH 14/61] assemble execute_trace process --- src/emysql_sup.erl | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/emysql_sup.erl b/src/emysql_sup.erl index 096abc68..2b8c55ea 100644 --- a/src/emysql_sup.erl +++ b/src/emysql_sup.erl @@ -32,7 +32,12 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init(_) -> - {ok, {{one_for_one, 10, 10}, [ - {emysql_statements, {emysql_statements, start_link, []}, permanent, 5000, worker, [emysql_statements]}, - {emysql_conn_mgr, {emysql_conn_mgr, start_link, []}, permanent, 5000, worker, [emysql_conn_mgr]} + {ok, {{one_for_one, 10, 10}, + [ + {emysql_statements, + {emysql_statements, start_link, []}, permanent, 5000, worker, [emysql_statements]}, + {emysql_conn_mgr, + {emysql_conn_mgr, start_link, []}, permanent, 5000, worker, [emysql_conn_mgr]}, + {execute_trace, + {execute_trace, start_link, []}, permanent, 5000, worker, [execute_trace]}, ]}}. From bce4aa3d2c82a39c01d04f2a179f991f6d1c0c38 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Wed, 30 Apr 2014 01:17:52 +0800 Subject: [PATCH 15/61] assemble execute_trace process --- src/emysql_sup.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emysql_sup.erl b/src/emysql_sup.erl index 2b8c55ea..7c3a7d5e 100644 --- a/src/emysql_sup.erl +++ b/src/emysql_sup.erl @@ -39,5 +39,5 @@ init(_) -> {emysql_conn_mgr, {emysql_conn_mgr, start_link, []}, permanent, 5000, worker, [emysql_conn_mgr]}, {execute_trace, - {execute_trace, start_link, []}, permanent, 5000, worker, [execute_trace]}, + {execute_trace, start_link, []}, permanent, 5000, worker, [execute_trace]} ]}}. From d2119b22232f74a13b8265e3e523d928a45c9e0c Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Wed, 30 Apr 2014 01:22:02 +0800 Subject: [PATCH 16/61] clean the code --- src/emysql_query.erl | 2 +- src/execute_trace.erl | 23 ++++++----------------- 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/src/emysql_query.erl b/src/emysql_query.erl index 507ad00c..15f70314 100644 --- a/src/emysql_query.erl +++ b/src/emysql_query.erl @@ -33,7 +33,7 @@ find(ConnOrPool, [RawSql | Values]) -> #emysql_connection{} = Conn -> mysql_conn:execute(Conn, RawSql, Values); Pool -> - emysql:execute(ConnPool, RawSql, Values) + emysql:execute(Pool, RawSql, Values) end. %%-------------------------------------------------------------------- diff --git a/src/execute_trace.erl b/src/execute_trace.erl index cb2c4539..93243525 100644 --- a/src/execute_trace.erl +++ b/src/execute_trace.erl @@ -13,6 +13,7 @@ %% API -export([prepare/2, unprepare/1, + execute/1, execute/2, begin_transaction/0, commit_transaction/0, @@ -63,9 +64,9 @@ unprepare(Name) -> execute(StmtName) -> execute(StmtName, []). execute(raw_query, Query) -> - gen_server:cast(?SERVER, {raw_query, Query}). + gen_server:cast(?SERVER, {raw_query, Query}); execute(StmtName, Params) -> - gen_server:cast(?SERVER, {execute, type_utils:any_to_binary(StmtName), Statement}). + gen_server:cast(?SERVER, {execute, type_utils:any_to_binary(StmtName), Params}). %%-------------------------------------------------------------------- %% @doc @@ -97,18 +98,6 @@ rollback_transaction() -> - -%%-------------------------------------------------------------------- -%% @doc -%% -%% @spec -%% @end -%%-------------------------------------------------------------------- -execute(StmtName, Statement) -> - gen_server:cast(?SERVER, {execute, StmtName, Statement}). - - - %%-------------------------------------------------------------------- %% @doc %% Starts the server @@ -171,14 +160,14 @@ handle_cast({prepare, Name, Statement}, #state{stmts = Stmts}=State) -> orddict:store(Name, [Statement, 1], Stmts); {ok, [Statement, Ref]} -> orddict:store(Name, [Statement, Ref + 1], Stmts); - {ok, [TheStatemnt, _]} -> + {ok, [TheStatement, _]} -> lager:error("Prepare statement (~p) conflicted: ~p <-> ~p", [Name, TheStatement, Statement]), Stmts end, {noreply, State#state{stmts = NStmts}}; -handle_cast({unprepare, Name}, #state{stmt = Stmts} = State) -> +handle_cast({unprepare, Name}, #state{stmts = Stmts} = State) -> NStmts = case orddict:find(Name, Stmts) of error -> Stmts; {ok, [Statement, 1]} -> @@ -194,7 +183,7 @@ handle_cast({execute, Name, Params}, #state{stmts = Stmts}=State) -> lager:warning("No statement (~p) found", [Name]); {ok, Statement} -> lager:debug("Execute ~p: ~p~nParams: ~p", - [Name, Statment, Params]) + [Name, Statement, Params]) end, {noreply, State}; From 6fd8918d51e62a6228de2c0ace9420d1a3fde89d Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Wed, 30 Apr 2014 09:29:52 +0800 Subject: [PATCH 17/61] add transaction trace --- src/emysql_conn.erl | 3 +++ src/execute_trace.erl | 14 +++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/emysql_conn.erl b/src/emysql_conn.erl index 3175d567..a182ea4c 100644 --- a/src/emysql_conn.erl +++ b/src/emysql_conn.erl @@ -168,12 +168,15 @@ unprepare(Connection, Name) -> send_recv(Connection, Packet). begin_transaction(Connection) -> + execute_trace:begin_transaction(), emysql_conn:execute(Connection, <<"BEGIN">>, []). rollback_transaction(Connection) -> + execute_trace:rollback_transaction(), emysql_conn:execute(Connection, <<"ROLLBACK">>, []). commit_transaction(Connection) -> + execute_trace:commit_transaction(), emysql_conn:execute(Connection, <<"COMMIT">>, []). open_n_connections(PoolId, N) -> diff --git a/src/execute_trace.erl b/src/execute_trace.erl index 93243525..7b482a8e 100644 --- a/src/execute_trace.erl +++ b/src/execute_trace.erl @@ -170,7 +170,7 @@ handle_cast({prepare, Name, Statement}, #state{stmts = Stmts}=State) -> handle_cast({unprepare, Name}, #state{stmts = Stmts} = State) -> NStmts = case orddict:find(Name, Stmts) of error -> Stmts; - {ok, [Statement, 1]} -> + {ok, [_Statement, 1]} -> orddict:erase(Name, Stmts); {ok, [Statement, Ref]} -> orddict:store(Name, [Statement, Ref - 1], Stmts) @@ -192,6 +192,18 @@ handle_cast({raw_query, Query}, #state{}=State) -> {noreply, State}; +handle_cast(begin_transaction, State) -> + lager:debug("BEGIN TRANSACTION"), + {noreply, State}; + +handle_cast(commit_transaction, State) -> + lager:debug("COMMIT"), + {noreply, State}; + +handle_cast(rollback_transaction, State) -> + lager:debug("ROLLBACK"), + {noreply, State}; + handle_cast(_Msg, State) -> {noreply, State}. From d6c4b505271f45632ba866a5a25b54dda8606dc3 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Wed, 30 Apr 2014 14:25:55 +0800 Subject: [PATCH 18/61] reformat execute log --- src/emysql_conn.erl | 2 +- src/execute_trace.erl | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/emysql_conn.erl b/src/emysql_conn.erl index a182ea4c..4cf0bad2 100644 --- a/src/emysql_conn.erl +++ b/src/emysql_conn.erl @@ -127,7 +127,7 @@ execute(Connection, Query, Args) when (is_list(Query) orelse is_binary(Query)) a OK when is_record(OK, ok_packet) -> ParamNamesBin = list_to_binary(string:join([[$@ | integer_to_list(I)] || I <- lists:seq(1, length(Args))], ", ")), % todo: utf8? Packet = <>, % todo: utf8? - execute_trace:execute(StmtName, ParamNamesBin), + execute_trace:execute(StmtName, Args), send_recv(Connection, Packet); Error -> Error diff --git a/src/execute_trace.erl b/src/execute_trace.erl index 7b482a8e..1e4e21e7 100644 --- a/src/execute_trace.erl +++ b/src/execute_trace.erl @@ -181,9 +181,9 @@ handle_cast({execute, Name, Params}, #state{stmts = Stmts}=State) -> case orddict:find(Name, Stmts) of error -> lager:warning("No statement (~p) found", [Name]); - {ok, Statement} -> - lager:debug("Execute ~p: ~p~nParams: ~p", - [Name, Statement, Params]) + {ok, [Statement, _Ref]} -> + lager:debug("Execute ~p: ~p", [Name, Statement]), + lager:debug("Params ~p: ~p", [Name, Params]) end, {noreply, State}; From 292d43b43fd9f59be794d3d216d3a5fe1917f6af Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Wed, 30 Apr 2014 14:26:14 +0800 Subject: [PATCH 19/61] complete find_each --- src/emysql_query.erl | 119 ++++++++++++++++++++++++++++--------------- 1 file changed, 78 insertions(+), 41 deletions(-) diff --git a/src/emysql_query.erl b/src/emysql_query.erl index 15f70314..8f0126c2 100644 --- a/src/emysql_query.erl +++ b/src/emysql_query.erl @@ -10,7 +10,8 @@ %% API -export([find/2, find/4]). --export([find_first/4, find_each/6]). +-export([find_first/4]). +-export([find_each/5, find_each/6]). -include("emysql.hrl"). @@ -47,14 +48,14 @@ find(ConnOrPool, [RawSql | Values]) -> %% @end %%-------------------------------------------------------------------- find(ConnOrPool, Table, SqlOptions, [Rec, RecFields] = _AsRec) -> - {FindSql, _, CondVals} = build_sql(Table, SqlOptions), + {FindSql, CondVals} = build_sql(Table, SqlOptions), Result = case ConnOrPool of #emysql_connection{} = Conn -> emysql_conn:execute(Conn, FindSql, CondVals); Pool -> emysql:execute(Pool, FindSql, CondVals) end, - emysql_util:as_record(Result, Rec, RecFields). + emysql_conv:as_record(Result, Rec, RecFields). %%-------------------------------------------------------------------- @@ -90,47 +91,40 @@ find_each(ConnOrPool, Table, SqlOptions, AsRec, Fun) -> find_each(ConnOrPool, Table, SqlOptions, 1000, AsRec, Fun). find_each(ConnOrPool, Table, SqlOptions, BatchSize, AsRec, Fun) -> - {FindSql, CountSql, CondVals} = build_sql(Table, SqlOptions), + BaseId = 0, + {[FindSql, FindCondVals], [CountSql, CountCondVals]} = build_sql(Table, SqlOptions, BatchSize, BaseId), + Result = case ConnOrPool of #emysql_connection{} = Conn -> - emysql_conn:execute(Conn, CountSql, CondVals); + emysql_conn:execute(Conn, CountSql, CountCondVals); Pool -> - emysql:execute(Pool, CountSql, CondVals) + emysql:execute(Pool, CountSql, CountCondVals) end, case Result of - #ok_packet{} -> - Total = 1000, - Limit = proplists:get_value(limit, SqlOptions, 0), - Remain = 10, % (Total > Limit ? Limit : Total), - do_find_each(ConnOrPool, Table, FindSql, CondVals, BatchSize, AsRec, Fun, Remain); - #error_packet{} -> + #result_packet{rows = [[Total]]} -> + Limit = proplists:get_value(limit, SqlOptions), + Remain = case {Limit, Total > Limit} of + {undefined, _} -> Total; + {_, true} -> Limit; + {_, false} -> Total + end, + do_find_each(ConnOrPool, Table, FindSql, FindCondVals, BatchSize, + AsRec, Fun, Remain, BaseId); + #error_packet{code = Code, msg = Msg} -> + throw({Code, Msg}); + _ -> ok end. - -do_find_each(ConnOrPool, Table, Sql, CondVals, BatchSize, [Rec, RecFields] = AsRec, Fun, Remain) -> - Result = case ConnOrPool of - #emysql_connection{} = Conn -> - emysql_conn:execute(Conn, Sql, CondVals); - Pool -> - emysql:execute(Pool, Sql, CondVals) - end, - lists:foreach( - fun(Item) -> - Fun(emysql_util:as_record(Result, Rec, RecFields)) - end, Result), - - case Remain - BatchSize > 0 of - true -> - do_find_each(ConnOrPool, Table, Sql, CondVals, BatchSize, AsRec, Fun, (Remain - BatchSize)); - false -> - ok - end. %%%=================================================================== %%% Internal functions %%%=================================================================== build_sql(Table, SqlOptions) -> + {Ret, _} = build_sql(Table, SqlOptions, undefined, undefined), + Ret. + +build_sql(Table, SqlOptions, BatchSize, BaseId) -> SelectFields = case proplists:get_value(select, SqlOptions) of undefined -> "*"; @@ -144,22 +138,65 @@ build_sql(Table, SqlOptions) -> [ Cond ] -> {"WHERE " ++ Cond, [ ]}; [Cond | Values] -> {"WHERE " ++ Cond, Values} end, + {Where2, CondVals2} = + case {BaseId, Where} of + {undefined, _} -> {Where, CondVals}; + {_, ""} -> {"WHERE id > ?", [BaseId]}; + _ -> + {Where ++ "AND id > ?", lists:append(CondVals, [BaseId])} + end, + Order = case proplists:get_value(order, SqlOptions) of undefined -> ""; - OrderBy -> " ORDER BY " ++ type_utils:any_to_list(OrderBy) - end, - Limit = case proplists:get_value(limit, SqlOptions) of - undefined -> ""; - [LV1, LV2] -> - " LIMIT " ++ type_utils:any_to_list(LV1) ++ ", " ++ type_utils:any_to_list(LV2); - LV3 -> " LIMIT " ++ type_utils:any_to_list(LV3) + OrderBy -> "ORDER BY " ++ type_utils:any_to_list(OrderBy) end, + + Limit = + case BatchSize of + undefined -> + case proplists:get_value(limit, SqlOptions) of + undefined -> ""; + [LV1, LV2] -> + "LIMIT " ++ type_utils:any_to_list(LV1) ++ ", " ++ type_utils:any_to_list(LV2); + LV3 -> "LIMIT " ++ type_utils:any_to_list(LV3) + end; + _ -> + "LIMIT " ++ type_utils:any_to_list(BatchSize) + end, + + Table2 = type_utils:any_to_list(Table), - FindSql = string:join(["SELECT", SelectFields, "FROM", Table2, Where, Order, Limit], " "), + FindSql = string:join(["SELECT", SelectFields, "FROM", Table2, Where2, Order, Limit], " "), CountSql = string:join(["SELECT COUNT(1) FROM", Table2, Where], " "), - {FindSql, CountSql, CondVals}. - + {[FindSql, CondVals2], [CountSql, CondVals]}. + + + +do_find_each(ConnOrPool, Table, Sql, CondVals, BatchSize, [Rec, RecFields] = AsRec, Fun, Remain, BaseId) -> + NCondVals = lists:append(lists_utils:droplast(CondVals), [BaseId]), + + Result = case ConnOrPool of + #emysql_connection{} = Conn -> + emysql_conn:execute(Conn, Sql, NCondVals); + Pool -> + emysql:execute(Pool, Sql, NCondVals) + end, + case Result of + #result_packet{rows = Rows} -> + emysql_conv:as_record(Result, Rec, RecFields, Fun), + LastRow = lists_utils:last(Rows), + [NextId | _Tail] = LastRow, + case Remain - BatchSize > 0 of + true -> + do_find_each(ConnOrPool, Table, Sql, CondVals, BatchSize, AsRec, + Fun, (Remain - BatchSize), NextId); + false -> + ok + end; + _ -> + failed + end. From a7863cf1ffda54f599ee4fdc2dd4149cfedce9b0 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Sat, 3 May 2014 23:06:25 +0800 Subject: [PATCH 20/61] add INPUT macro --- include/emysql.hrl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/include/emysql.hrl b/include/emysql.hrl index 22f08d3c..654352b1 100644 --- a/include/emysql.hrl +++ b/include/emysql.hrl @@ -181,5 +181,13 @@ -define(SERVER_STATUS_METADATA_CHANGED, 1024). % Wrap the query result in erlang record --define(AS_REC(Rec), [Rec, record_info(fields, Rec)]). +-define(AS_REC(Rec), [Rec, record_info(fields, Rec)]). +-define(INPUT(Records), ((fun() -> + case Records of + undefined -> undefined; + Rec when is_record(Rec) -> + [Rec, record_info(fields, Rec)]; + [Rec | _] -> + [Records, record_info(fields, Rec)] + end)()) ). From e765c98027f11f00bb68f2e080b8888278acd00d Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Sat, 3 May 2014 23:17:49 +0800 Subject: [PATCH 21/61] emysql_saver module wrappes some update/save functions --- src/emysql_saver.erl | 112 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 src/emysql_saver.erl diff --git a/src/emysql_saver.erl b/src/emysql_saver.erl new file mode 100644 index 00000000..1874b21a --- /dev/null +++ b/src/emysql_saver.erl @@ -0,0 +1,112 @@ +%%%------------------------------------------------------------------- +%%% @author Jack Tang +%%% @copyright (C) 2014, Jack Tang +%%% @doc +%%% +%%% @end +%%% Created : 28 Apr 2014 by Jack Tang +%%%------------------------------------------------------------------- +-module(emysql_saver). + +%% API +-export([save/3, save/4]). +-export([find_or_create_by/4]). + +-include("emysql.hrl"). +%%%=================================================================== +%%% API +%%%=================================================================== + +%%-------------------------------------------------------------------- +%% @doc +%% Options = [{insert_ignore, true}, +%% {batch_size, 1000}, +%% {update_attr, [data] }] +%% emysql_saver:save(pool, test, ?INPUT(#test{data = "hello"}, Options) +%% +%% @spec +%% @end +%%-------------------------------------------------------------------- +save(_ConnOrPool, _Table, undefined) -> + ok. +save(ConnOrPool, Table, RecordInput) -> + save(ConnOrPool, Table, RecordInput, []). + +save(ConnOrPool, Table, [Records, Fields] = _RecordInput, Options) -> + {InsertOrUpdate, Sql, Values} = build_sql(Table, Records, Fields, Options), + Result = case ConnOrPool of + #emysql_connection{} = Conn -> + emysql_conn:execute(Conn, Sql, Values); + Pool -> + emysql:execute(Pool, Sql, Values) + end, + case {InsertOrUpdate, Result} of + {insert, #ok_packet{affected_rows = Affected, insert_id = InsertId}} -> + case {Affected, length(Records), lists:nth(1, Fields)} of + {1, 1, id} -> + [Record] = Records, + setelement(2, Record, InsertId), + Record; + _ -> ok + end; + _ -> Records + end, + + ok. + +%%-------------------------------------------------------------------- +%% @doc +%% +%% emysql_saver:find_or_create_by(pool, test, ) +%% +%% @spec +%% @end +%%-------------------------------------------------------------------- +find_or_create_by(ConnOrPool, Table, FindCond, CreateFun) -> + [ConnOrPool, Table, FindCond, CreateFun], + ok. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== +%% insert into table(f1, f2, f3) values(v1, v2, v3); +%% insert into table(f1, f2, f3) values(v11, v12, v13), (v21, v22, v23), ...; +%% update table set f1 = ? +build_sql(Table, [Record | Tail] = _Records, Fields, Options) -> + + UpdateFields = case proplists:get_value(update_fields, Options) of + undefined -> []; + V -> V + end, + + {_, UpdateFields, UpdateFIndex, UpdateVals} = + lists:foldl( + fun(Field, [Index, EffectedFields, EffectedIndex, Vals]) -> + case lists:member(Field, UpdateFields) of + true -> + NEffectedFields = [Field | EffectedFields], + NEffectedIndex = [Index | EffectedIndex], + Val = element(Index + 1, Record), + NVals = [Val | Vals], + {Index + 1, NEffectedFields, NEffectedIndex, NVals}; + false -> + {Index + 1, EffectedFields, EffectedIndex, Vals} + end + end, {1, [], [], []}, Fields), + + case element(2, Record) of + undefined -> % insert + SqlHead = "INSERT INTO " ++ type_utils:any_to_list(Table), + SqlFields = string:join(UpdateFields, ","), + SqlValues = string:join(lists:duplicate(length(UpdateFields), "?"), ","), + Sql = string:join([SqlHead, "(", SqlFields, ") VALUES (", SqlValues, ")"], " "), + {insert, Sql, UpdateVals}; + _ -> % update + PK = type_utils:any_to_list(lists:nth(1, Fields)), + SqlHead = "UPDATE " ++ type_utils:any_to_list(Table), + SqlTail = "WHERE " ++ PK ++ " = ?", + SqlSet = string:join([ F ++ " = ?" || F <- UpdateFields], ","), + Sql = string:join([SqlHead, "SET ", SqlSet, SqlTail], " "), + {update, Sql, lists:append(UpdateVals, [element(2, Record)])} + end. + From 14d43afa2d083c04e841a932a173b707e66a6545 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Sun, 4 May 2014 17:14:06 +0800 Subject: [PATCH 22/61] add batch save/update --- src/emysql_saver.erl | 187 +++++++++++++++++++++++++++++++------------ 1 file changed, 135 insertions(+), 52 deletions(-) diff --git a/src/emysql_saver.erl b/src/emysql_saver.erl index 1874b21a..4f12de66 100644 --- a/src/emysql_saver.erl +++ b/src/emysql_saver.erl @@ -19,40 +19,58 @@ %%-------------------------------------------------------------------- %% @doc -%% Options = [{insert_ignore, true}, -%% {batch_size, 1000}, -%% {update_attr, [data] }] +%% Options = [{insert_ignore, true}, +%% {batch_size, 1000}, +%% {update_attrs, [data] }] %% emysql_saver:save(pool, test, ?INPUT(#test{data = "hello"}, Options) %% %% @spec %% @end %%-------------------------------------------------------------------- save(_ConnOrPool, _Table, undefined) -> - ok. + ok; +save(_ConnOrPool, _Table, [[], _]) -> + ok; save(ConnOrPool, Table, RecordInput) -> - save(ConnOrPool, Table, RecordInput, []). + DefOpt = [{auto_id, true}, + {batch_size, 1000}], + save(ConnOrPool, Table, RecordInput, DefOpt). save(ConnOrPool, Table, [Records, Fields] = _RecordInput, Options) -> - {InsertOrUpdate, Sql, Values} = build_sql(Table, Records, Fields, Options), - Result = case ConnOrPool of - #emysql_connection{} = Conn -> - emysql_conn:execute(Conn, Sql, Values); - Pool -> - emysql:execute(Pool, Sql, Values) - end, - case {InsertOrUpdate, Result} of - {insert, #ok_packet{affected_rows = Affected, insert_id = InsertId}} -> - case {Affected, length(Records), lists:nth(1, Fields)} of - {1, 1, id} -> - [Record] = Records, - setelement(2, Record, InsertId), - Record; - _ -> ok + + case build_sql(Table, Records, Fields, Options) of + {insert, Sqls} -> + lists:foldl( + X = fun({Sql, Values}, AccIn) -> + Result = case ConnOrPool of + #emysql_connection{} = Conn -> + emysql_conn:execute(Conn, Sql, Values); + Pool -> + emysql:execute(Pool, Sql, Values) + end, + %case Result of + % #ok_packat{effect} + %case {Affected, length(Records), lists:nth(1, Fields)} of + % {1, 1, id} -> + % [Record] = Records, + % [setelement(2, Record, InsertId) | AccIn]; + % _ -> AccIn + %end + ok + end, [], Sqls), + case X of + [] -> ok; + [V] -> V end; - _ -> Records - end, - - ok. + {update, {Sql, Values} } -> + Result = case ConnOrPool of + #emysql_connection{} = Conn -> + emysql_conn:execute(Conn, Sql, Values); + Pool -> + emysql:execute(Pool, Sql, Values) + end, + ok + end. %%-------------------------------------------------------------------- %% @doc @@ -72,41 +90,106 @@ find_or_create_by(ConnOrPool, Table, FindCond, CreateFun) -> %% insert into table(f1, f2, f3) values(v1, v2, v3); %% insert into table(f1, f2, f3) values(v11, v12, v13), (v21, v22, v23), ...; %% update table set f1 = ? -build_sql(Table, [Record | Tail] = _Records, Fields, Options) -> +build_sql(_Table, [] = _Records, _Fields, _Options) -> + {error, no_input}; +build_sql(Table, [Record | _Tail] = Records, Fields, Options) -> + BatchSize = proplists:get_value(batch_size, Options, 1000), + + {UpdateFields, UpdateFIndex, UpdateVals} = select_update_fields_index(Record, Fields, Options), + + case element(2, Record) of + undefined -> % insert + SqlAndValues = generate_insert_sql(Table, UpdateFields, UpdateFIndex, Records, BatchSize), + {insert, SqlAndValues}; + _ -> % update + [PK | _] = Fields, + PKPair = {type_utils:any_to_list(PK), element(2, Record)}, + SqlAndValues = generate_update_sql(Table, UpdateFields, UpdateVals, PKPair), + {update, SqlAndValues} + end. + +select_update_fields_index(Record, Fields, Options) -> + {UAOptIsSet, UpdateAttrs} = + case proplists:get_value(update_attrs, Options) of + undefined -> {no, []}; + V -> {yes, V} + end, - UpdateFields = case proplists:get_value(update_fields, Options) of - undefined -> []; - V -> V - end, + AutoId = case proplists:get_value(auto_id, Options) of + true -> true; + _ -> false + end, - {_, UpdateFields, UpdateFIndex, UpdateVals} = + {_, UpdateFields, UpdateFIndex, UpdateVals} = lists:foldl( - fun(Field, [Index, EffectedFields, EffectedIndex, Vals]) -> - case lists:member(Field, UpdateFields) of - true -> - NEffectedFields = [Field | EffectedFields], - NEffectedIndex = [Index | EffectedIndex], + fun(Field, {Index, EffectedFields, EffectedIndex, Vals}) -> + case {Field, AutoId, UAOptIsSet, lists:member(Field, UpdateAttrs)} of + {id, true, _, _} -> + {Index + 1, EffectedFields, EffectedIndex, Vals}; + {_, _, no, _} -> + NEffectedFields = [type_utils:any_to_list(Field) | EffectedFields], + NEffectedIndex = [(Index + 1) | EffectedIndex], + Val = element(Index + 1, Record), + {Index + 1, NEffectedFields, NEffectedIndex, [Val | Vals]}; + {_, _, yes, true} -> + NEffectedFields = [type_utils:any_to_list(Field) | EffectedFields], + NEffectedIndex = [(Index + 1) | EffectedIndex], Val = element(Index + 1, Record), - NVals = [Val | Vals], - {Index + 1, NEffectedFields, NEffectedIndex, NVals}; - false -> + {Index + 1, NEffectedFields, NEffectedIndex, [Val | Vals]}; + {_, _, yes, false} -> {Index + 1, EffectedFields, EffectedIndex, Vals} end end, {1, [], [], []}, Fields), + {UpdateFields, UpdateFIndex, UpdateVals}. - case element(2, Record) of - undefined -> % insert - SqlHead = "INSERT INTO " ++ type_utils:any_to_list(Table), - SqlFields = string:join(UpdateFields, ","), - SqlValues = string:join(lists:duplicate(length(UpdateFields), "?"), ","), - Sql = string:join([SqlHead, "(", SqlFields, ") VALUES (", SqlValues, ")"], " "), - {insert, Sql, UpdateVals}; - _ -> % update - PK = type_utils:any_to_list(lists:nth(1, Fields)), - SqlHead = "UPDATE " ++ type_utils:any_to_list(Table), - SqlTail = "WHERE " ++ PK ++ " = ?", - SqlSet = string:join([ F ++ " = ?" || F <- UpdateFields], ","), - Sql = string:join([SqlHead, "SET ", SqlSet, SqlTail], " "), - {update, Sql, lists:append(UpdateVals, [element(2, Record)])} +generate_insert_sql(Table, UpdateFields, UpdateFIndex, Records, BatchSize) -> + SqlHead = "INSERT INTO " ++ type_utils:any_to_list(Table), + SqlFields = string:join(UpdateFields, ","), + ValuesInSql = "(" ++ string:join(lists:duplicate(length(UpdateFields), "?"), ",") ++ ")", + + BatchCount = length(Records) div BatchSize, + BatchRemainSize = length(Records) rem BatchSize, + + Batch1Sql = case BatchCount of + 0 -> undefined; + _ -> + SqlValues1 = string:join(lists:duplicate(BatchSize, ValuesInSql), ","), + string:join([SqlHead, "(", SqlFields, ") VALUES ", SqlValues1], " ") + end, + Batch2Sql = case BatchRemainSize of + 0 -> undefined; + _ -> + SqlValues2 = string:join(lists:duplicate(BatchRemainSize, ValuesInSql), ","), + string:join([SqlHead, "(", SqlFields, ") VALUES ", SqlValues2], " ") + end, + + RecBatchValues = + lists_utils:split(BatchSize, Records, + fun(NRecords) -> + lists:foldl( + fun(RecItem, AccIn) -> + lists:foldl( + fun(Idx, AccIn2) -> + Val = element(Idx, RecItem), + [Val | AccIn2] + end, AccIn, UpdateFIndex) + end, [], NRecords) + end), + case BatchRemainSize of + 0 -> [{Batch1Sql, lists:merge(RecBatchValues)}]; + _ -> + {Batch1Values, Batch2Values} = lists:split(BatchCount, RecBatchValues), + case Batch1Sql of + undefined -> [{Batch2Sql, lists:merge(Batch2Values)}]; + _ -> [{Batch1Sql, lists:merge(Batch1Values)}, + {Batch2Sql, lists:merge(Batch2Values)}] + end end. +generate_update_sql(Table, UpdateFields, UpdateVals, {FieldPK, PKVal}) -> + SqlHead = "UPDATE " ++ type_utils:any_to_list(Table), + SqlTail = "WHERE " ++ FieldPK ++ " = ?", + SqlSet = string:join([ F ++ " = ?" || F <- UpdateFields], ","), + Sql = string:join([SqlHead, "SET ", SqlSet, SqlTail], " "), + + {Sql, lists:append(UpdateVals, [PKVal])}. From 7c8380628e4e4728149640bd05b2d99389038ea4 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Mon, 5 May 2014 00:20:23 +0800 Subject: [PATCH 23/61] add find_and_create_by --- src/emysql_saver.erl | 58 ++++++++++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/src/emysql_saver.erl b/src/emysql_saver.erl index 4f12de66..b10d701b 100644 --- a/src/emysql_saver.erl +++ b/src/emysql_saver.erl @@ -37,52 +37,64 @@ save(ConnOrPool, Table, RecordInput) -> save(ConnOrPool, Table, RecordInput, DefOpt). save(ConnOrPool, Table, [Records, Fields] = _RecordInput, Options) -> - + [FieldPK | _ ] = Fields, case build_sql(Table, Records, Fields, Options) of {insert, Sqls} -> - lists:foldl( - X = fun({Sql, Values}, AccIn) -> + X = lists:foldl( + fun({Sql, Values}, AccIn) -> Result = case ConnOrPool of #emysql_connection{} = Conn -> emysql_conn:execute(Conn, Sql, Values); Pool -> emysql:execute(Pool, Sql, Values) end, - %case Result of - % #ok_packat{effect} - %case {Affected, length(Records), lists:nth(1, Fields)} of - % {1, 1, id} -> - % [Record] = Records, - % [setelement(2, Record, InsertId) | AccIn]; - % _ -> AccIn - %end - ok + case Result of + #ok_packet{affected_rows = Affected, insert_id = InsertId} -> + case {Affected, length(Records), FieldPK} of + {1, 1, id} -> + [Record] = Records, + [setelement(2, Record, InsertId) | AccIn]; + _ -> AccIn + end; + #error_packet{code = Code, msg = Msg} -> + throw({Code, Msg}) + end end, [], Sqls), case X of [] -> ok; [V] -> V end; {update, {Sql, Values} } -> - Result = case ConnOrPool of - #emysql_connection{} = Conn -> - emysql_conn:execute(Conn, Sql, Values); - Pool -> - emysql:execute(Pool, Sql, Values) - end, - ok + case ConnOrPool of + #emysql_connection{} = Conn -> + emysql_conn:execute(Conn, Sql, Values); + Pool -> + emysql:execute(Pool, Sql, Values) + end end. %%-------------------------------------------------------------------- %% @doc %% -%% emysql_saver:find_or_create_by(pool, test, ) +%% emysql_saver:find_or_create_by(pool, test, ["select * from test where data = ?", "hello"], +%% fun() -> +%% ?INPUT(#test{data = "find me"}) +%% end) %% %% @spec %% @end %%-------------------------------------------------------------------- -find_or_create_by(ConnOrPool, Table, FindCond, CreateFun) -> - [ConnOrPool, Table, FindCond, CreateFun], - ok. +find_or_create_by(ConnOrPool, Table, FindSql, CreateFun) -> + Result = emysql_query:find(ConnOrPool, FindSql), + [[Record], Fields] = CreateFun(), + case Result of + #result_packet{rows = Rows} when length(Rows) =:= 0 -> + save(ConnOrPool, Table, [[Record], Fields]); + #result_packet{} -> + emysql_conv:as_record(Result, element(1, Record), Fields); + Other -> + Other + end. %%%=================================================================== %%% Internal functions From 36a62a043cf6e5670db7cba0c1fa1f18c0bada2b Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Mon, 5 May 2014 23:56:24 +0800 Subject: [PATCH 24/61] add rebar deps notes --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index b603ba72..3f100acb 100644 --- a/README.md +++ b/README.md @@ -284,6 +284,18 @@ or (after building emysql.app and the database, as explained above), start a_hel General Notes on using Emysql, including the actual specs: +. Resolve Emysql dependency via rebar.config if possible + +```Erlang +{deps, +[ {'emysql', ".*", { git, "git://github.com/Eonblast/Emysql.git", "master" } } ] +``` +or some other branch +```Erlang +{deps, +[ {'emysql', ".*", { git, "git://github.com/jacktang/Emysql.git", "taodi" } } ] +``` + #### Starting an Application The Emysql driver is an Erlang gen-server, and, application. From aede183f4aea4adb9300edbd5803c940b2567ef8 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Mon, 5 May 2014 23:58:35 +0800 Subject: [PATCH 25/61] format it --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3f100acb..ac5d4fc5 100644 --- a/README.md +++ b/README.md @@ -284,7 +284,7 @@ or (after building emysql.app and the database, as explained above), start a_hel General Notes on using Emysql, including the actual specs: -. Resolve Emysql dependency via rebar.config if possible +#### Resolve Emysql dependency via rebar.config if possible ```Erlang {deps, From 76ee47ab50290f949e7a0c918ed2ea1d86822388 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Tue, 6 May 2014 14:12:03 +0800 Subject: [PATCH 26/61] add lager lib --- rebar.config | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/rebar.config b/rebar.config index 789878a9..b21cb518 100644 --- a/rebar.config +++ b/rebar.config @@ -1,17 +1,20 @@ % -*- Erlang -*- % vim: ts=4 sw=4 et ft=erlang {erl_opts, [ - nowarn_deprecated_type + nowarn_deprecated_type, + {parse_transform, lager_transform} ]}. + {pre_hooks,[ {"linux|bsd|darwin|solaris", compile, "escript ./support/crypto_compat.escript"}, {"win32", compile, "escript.exe support/crypto_compat.escript"} ]}. {deps, [ - {'espec', ".*", { git, "git://github.com/lucaspiller/espec.git", "master" } }, - {'erl_utils', ".*", { git, "git://github.com/jacktang/erl_utils.git", "master"} } - ]}. + {'espec', ".*", { git, "git://github.com/lucaspiller/espec.git", "master" } }, + {'lager', ".*", { git, "git://github.com/basho/lager.git", "master" } }, + {'erl_utils', ".*", { git, "git://github.com/jacktang/erl_utils.git", "master" } } +]}. {xref_warnings, true}. From 87b0a4ead60d02073e05b75c0130904e82d2633a Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Tue, 6 May 2014 17:52:36 +0800 Subject: [PATCH 27/61] fix typo --- src/emysql_query.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emysql_query.erl b/src/emysql_query.erl index 8f0126c2..6da91607 100644 --- a/src/emysql_query.erl +++ b/src/emysql_query.erl @@ -32,7 +32,7 @@ find(ConnOrPool, [RawSql | Values]) -> case ConnOrPool of #emysql_connection{} = Conn -> - mysql_conn:execute(Conn, RawSql, Values); + emysql_conn:execute(Conn, RawSql, Values); Pool -> emysql:execute(Pool, RawSql, Values) end. From 170529ece5c8ea0ca28f3d13308b79a5c90a8bb4 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Wed, 7 May 2014 10:09:51 +0800 Subject: [PATCH 28/61] support dict style 'where' --- src/emysql_query.erl | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/emysql_query.erl b/src/emysql_query.erl index 6da91607..736a93b4 100644 --- a/src/emysql_query.erl +++ b/src/emysql_query.erl @@ -43,7 +43,11 @@ find(ConnOrPool, [RawSql | Values]) -> %% {where, ["age > ?", Age]}, %% {order, "id desc"}, %% {limit, 100 }], ?AS_REC(user)) -%% +%% or +%% find(my_pool, users, [ {select, [name, age]}, +%% {where, [{age, '>', Age}]}, +%% {order, "id desc"}, +%% {limit, 100 }], ?AS_REC(user)) %% @spec %% @end %%-------------------------------------------------------------------- @@ -135,6 +139,27 @@ build_sql(Table, SqlOptions, BatchSize, BaseId) -> {Where, CondVals} = case proplists:get_value(where, SqlOptions) of undefined -> {"", [ ]}; [ ] -> {"", [ ]}; + [ Head | _T] = L when is_tuple(Head) -> + {Stmt, Vals} = + lists:foldl( + fun(Item, {StmtAcc, ValAcc}) -> + case Item of + {K, between, [V1, V2]} -> + S = type_utils:any_to_list(K) ++ " BETWEEN ? AND ? ", + {[S | StmtAcc], [V1, V2 | ValAcc]}; + {K, OP, V3} -> + S = string:join([type_utils:any_to_list(K), + type_utils:any_to_list(OP), + "?"], " "), + {[S | StmtAcc], [V3 | ValAcc] }; + {K, V4} -> + S = type_utils:any_to_list(K) ++ " = ?", + {[S | StmtAcc], [V4 | ValAcc]}; + _ -> + {StmtAcc, ValAcc} + end + end, {[], []}, L), + {string:join(Stmt, " AND "), Vals}; [ Cond ] -> {"WHERE " ++ Cond, [ ]}; [Cond | Values] -> {"WHERE " ++ Cond, Values} end, From a5087ea81b3f55f9924d132b3303a35ffed7c680 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Wed, 7 May 2014 14:36:59 +0800 Subject: [PATCH 29/61] add AS_VAL support --- include/emysql.hrl | 3 ++- src/emysql_query.erl | 53 +++++++++++++++++++++++++++++++------------- 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/include/emysql.hrl b/include/emysql.hrl index 654352b1..d860c687 100644 --- a/include/emysql.hrl +++ b/include/emysql.hrl @@ -182,6 +182,8 @@ % Wrap the query result in erlang record -define(AS_REC(Rec), [Rec, record_info(fields, Rec)]). +-define(AS_VAL, as_val). +% -define(INPUT(Records), ((fun() -> case Records of undefined -> undefined; @@ -190,4 +192,3 @@ [Rec | _] -> [Records, record_info(fields, Rec)] end)()) ). - diff --git a/src/emysql_query.erl b/src/emysql_query.erl index 736a93b4..2728a58c 100644 --- a/src/emysql_query.erl +++ b/src/emysql_query.erl @@ -9,7 +9,7 @@ -module(emysql_query). %% API --export([find/2, find/4]). +-export([find/2, find/3, find/4]). -export([find_first/4]). -export([find_each/5, find_each/6]). @@ -39,32 +39,51 @@ find(ConnOrPool, [RawSql | Values]) -> %%-------------------------------------------------------------------- %% @doc -%% e.g: find(my_pool, users, [ {select, [name, age]}, +%% e.g: find(my_pool, users, [ {select, ["name", "age"]}, %% {where, ["age > ?", Age]}, %% {order, "id desc"}, %% {limit, 100 }], ?AS_REC(user)) %% or -%% find(my_pool, users, [ {select, [name, age]}, +%% +%% find(my_pool, users, [ {select, ["name", "age"]}, %% {where, [{age, '>', Age}]}, %% {order, "id desc"}, %% {limit, 100 }], ?AS_REC(user)) +%% +%% or +%% +%% find(my_pool, users, [ {select, ["count(unique name)"]}, +%% {where, [{age, '>', Age}]}, +%% {order, "id desc"}, +%% {limit, 100 }], ?AS_VAL) +%% +%% %% @spec %% @end %%-------------------------------------------------------------------- -find(ConnOrPool, Table, SqlOptions, [Rec, RecFields] = _AsRec) -> +find(ConnOrPool, Table, SqlOptions) -> {FindSql, CondVals} = build_sql(Table, SqlOptions), - Result = case ConnOrPool of - #emysql_connection{} = Conn -> - emysql_conn:execute(Conn, FindSql, CondVals); - Pool -> - emysql:execute(Pool, FindSql, CondVals) - end, + case ConnOrPool of + #emysql_connection{} = Conn -> + emysql_conn:execute(Conn, FindSql, CondVals); + Pool -> + emysql:execute(Pool, FindSql, CondVals) + end. +find(ConnOrPool, Table, SqlOptions, ?AS_VAL) -> + case find(ConnOrPool, Table, SqlOptions) of + #result_packet{rows = [[R] ]} -> R; + #result_packet{rows = [ Rs ]} -> Rs; + Other -> Other + end; + +find(ConnOrPool, Table, SqlOptions, [Rec, RecFields] = _AsRec) -> + Result = find(ConnOrPool, Table, SqlOptions), emysql_conv:as_record(Result, Rec, RecFields). %%-------------------------------------------------------------------- %% @doc -%% e.g: find(my_pool, users, [ {select, [name, age]}, +%% e.g: find(my_pool, users, [ {select, ["name", "age"]}, %% {where, ["age > ?", Age]}, %% {order, "id desc"} ], ?AS_REC(user)) %% @@ -96,7 +115,9 @@ find_each(ConnOrPool, Table, SqlOptions, AsRec, Fun) -> find_each(ConnOrPool, Table, SqlOptions, BatchSize, AsRec, Fun) -> BaseId = 0, - {[FindSql, FindCondVals], [CountSql, CountCondVals]} = build_sql(Table, SqlOptions, BatchSize, BaseId), + + {[FindSql, FindCondVals], + [CountSql, CountCondVals]} = build_sql(Table, SqlOptions, BatchSize, BaseId), Result = case ConnOrPool of #emysql_connection{} = Conn -> @@ -125,8 +146,8 @@ find_each(ConnOrPool, Table, SqlOptions, BatchSize, AsRec, Fun) -> %%% Internal functions %%%=================================================================== build_sql(Table, SqlOptions) -> - {Ret, _} = build_sql(Table, SqlOptions, undefined, undefined), - Ret. + {[FindSql, FindCondVals], _} = build_sql(Table, SqlOptions, undefined, undefined), + {FindSql, FindCondVals}. build_sql(Table, SqlOptions, BatchSize, BaseId) -> SelectFields = @@ -159,8 +180,8 @@ build_sql(Table, SqlOptions, BatchSize, BaseId) -> {StmtAcc, ValAcc} end end, {[], []}, L), - {string:join(Stmt, " AND "), Vals}; - [ Cond ] -> {"WHERE " ++ Cond, [ ]}; + {"WHERE " ++ string:join(Stmt, " AND "), Vals}; + [ Cond ] -> {"WHERE " ++ Cond, [ ]}; [Cond | Values] -> {"WHERE " ++ Cond, Values} end, {Where2, CondVals2} = From 0de87199925d9ee22f900c4e6ea0f91bb249fcb9 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Thu, 8 May 2014 00:31:10 +0800 Subject: [PATCH 30/61] supports insert ignore --- src/emysql_saver.erl | 42 +++++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/src/emysql_saver.erl b/src/emysql_saver.erl index b10d701b..1f549b35 100644 --- a/src/emysql_saver.erl +++ b/src/emysql_saver.erl @@ -22,8 +22,12 @@ %% Options = [{insert_ignore, true}, %% {batch_size, 1000}, %% {update_attrs, [data] }] -%% emysql_saver:save(pool, test, ?INPUT(#test{data = "hello"}, Options) -%% +%% emysql_saver:save(pool, test, ?INPUT(#test{data = "hello"}, Options). +%% +%% or +%% +%% emysql_saver:save(pool, test, ?INPUT(#test{data = "hello"})). +%% %% @spec %% @end %%-------------------------------------------------------------------- @@ -86,14 +90,18 @@ save(ConnOrPool, Table, [Records, Fields] = _RecordInput, Options) -> %%-------------------------------------------------------------------- find_or_create_by(ConnOrPool, Table, FindSql, CreateFun) -> Result = emysql_query:find(ConnOrPool, FindSql), - [[Record], Fields] = CreateFun(), - case Result of - #result_packet{rows = Rows} when length(Rows) =:= 0 -> - save(ConnOrPool, Table, [[Record], Fields]); - #result_packet{} -> - emysql_conv:as_record(Result, element(1, Record), Fields); - Other -> - Other + case CreateFun() of + [[Record], Fields] -> + case Result of + #result_packet{rows = Rows} when length(Rows) =:= 0 -> + save(ConnOrPool, Table, [[Record], Fields]); + #result_packet{} -> + emysql_conv:as_record(Result, element(1, Record), Fields); + Other -> + Other + end; + _ -> + io:format("Forget to wrap return value using ?INPUT?", []) end. %%%=================================================================== @@ -105,13 +113,11 @@ find_or_create_by(ConnOrPool, Table, FindSql, CreateFun) -> build_sql(_Table, [] = _Records, _Fields, _Options) -> {error, no_input}; build_sql(Table, [Record | _Tail] = Records, Fields, Options) -> - BatchSize = proplists:get_value(batch_size, Options, 1000), - {UpdateFields, UpdateFIndex, UpdateVals} = select_update_fields_index(Record, Fields, Options), case element(2, Record) of undefined -> % insert - SqlAndValues = generate_insert_sql(Table, UpdateFields, UpdateFIndex, Records, BatchSize), + SqlAndValues = generate_insert_sql(Table, UpdateFields, UpdateFIndex, Records, Options), {insert, SqlAndValues}; _ -> % update [PK | _] = Fields, @@ -154,8 +160,14 @@ select_update_fields_index(Record, Fields, Options) -> end, {1, [], [], []}, Fields), {UpdateFields, UpdateFIndex, UpdateVals}. -generate_insert_sql(Table, UpdateFields, UpdateFIndex, Records, BatchSize) -> - SqlHead = "INSERT INTO " ++ type_utils:any_to_list(Table), +generate_insert_sql(Table, UpdateFields, UpdateFIndex, Records, Options) -> + BatchSize = proplists:get_value(batch_size, Options, 1000), + SqlHead = case proplists:get_value(insert_ignore, Options, false) of + true -> + "INSERT IGNORE INTO " ++ type_utils:any_to_list(Table); + _ -> + "INSERT INTO " ++ type_utils:any_to_list(Table) + end, SqlFields = string:join(UpdateFields, ","), ValuesInSql = "(" ++ string:join(lists:duplicate(length(UpdateFields), "?"), ",") ++ ")", From 922c3e689e93f81f65b1fb9604801ace948892a6 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Thu, 8 May 2014 12:07:43 +0800 Subject: [PATCH 31/61] fix typos --- include/emysql.hrl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/emysql.hrl b/include/emysql.hrl index d860c687..c78091a2 100644 --- a/include/emysql.hrl +++ b/include/emysql.hrl @@ -191,4 +191,5 @@ [Rec, record_info(fields, Rec)]; [Rec | _] -> [Records, record_info(fields, Rec)] - end)()) ). + end + end)()) ). From 65182a96cfc11e51ca22c759de106e64e84ecf1a Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Thu, 8 May 2014 14:14:14 +0800 Subject: [PATCH 32/61] refine ?INPUT macro --- include/emysql.hrl | 15 ++++++++++----- src/emysql_saver.erl | 4 ++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/include/emysql.hrl b/include/emysql.hrl index c78091a2..6878890f 100644 --- a/include/emysql.hrl +++ b/include/emysql.hrl @@ -186,10 +186,15 @@ % -define(INPUT(Records), ((fun() -> case Records of - undefined -> undefined; - Rec when is_record(Rec) -> - [Rec, record_info(fields, Rec)]; - [Rec | _] -> - [Records, record_info(fields, Rec)] + undefined -> {error, invalid_input}; + Rec when is_tuple(Rec) -> + RecAtom = element(1, Rec), + [Rec, record_info(fields, RecAtom)]; + [Rec | _] when is_tuple(Rec) -> + RecAtom = element(1, Rec), + [Records, record_info(fields, RecAtom)]; + _ -> + io:format("Can't handle the input: ~p~n", [Records]), + {error, invalid_input} end end)()) ). diff --git a/src/emysql_saver.erl b/src/emysql_saver.erl index 1f549b35..4cc00718 100644 --- a/src/emysql_saver.erl +++ b/src/emysql_saver.erl @@ -35,11 +35,15 @@ save(_ConnOrPool, _Table, undefined) -> ok; save(_ConnOrPool, _Table, [[], _]) -> ok; +save(_ConnOrPool, _Table, {error, Reason}) -> + {error, Reason}; save(ConnOrPool, Table, RecordInput) -> DefOpt = [{auto_id, true}, {batch_size, 1000}], save(ConnOrPool, Table, RecordInput, DefOpt). +save(_ConnOrPool, _Table, {error, Reason}, _Options) -> + {error, Reason}; save(ConnOrPool, Table, [Records, Fields] = _RecordInput, Options) -> [FieldPK | _ ] = Fields, case build_sql(Table, Records, Fields, Options) of From 259a8afa0f7311e36a4ed77ff338e6792464679b Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Thu, 8 May 2014 15:13:13 +0800 Subject: [PATCH 33/61] fix INPUT macro --- include/emysql.hrl | 17 ++--------------- src/emysql_saver.erl | 10 +++++----- 2 files changed, 7 insertions(+), 20 deletions(-) diff --git a/include/emysql.hrl b/include/emysql.hrl index 6878890f..70e1f81c 100644 --- a/include/emysql.hrl +++ b/include/emysql.hrl @@ -181,20 +181,7 @@ -define(SERVER_STATUS_METADATA_CHANGED, 1024). % Wrap the query result in erlang record --define(AS_REC(Rec), [Rec, record_info(fields, Rec)]). +-define(AS_REC(RecAtom), [Rec, record_info(fields, RecAtom)]). -define(AS_VAL, as_val). % --define(INPUT(Records), ((fun() -> - case Records of - undefined -> {error, invalid_input}; - Rec when is_tuple(Rec) -> - RecAtom = element(1, Rec), - [Rec, record_info(fields, RecAtom)]; - [Rec | _] when is_tuple(Rec) -> - RecAtom = element(1, Rec), - [Records, record_info(fields, RecAtom)]; - _ -> - io:format("Can't handle the input: ~p~n", [Records]), - {error, invalid_input} - end - end)()) ). +-define(INPUT(RecAtom, Records), [Records, record_info(fields, RecAtom)]). diff --git a/src/emysql_saver.erl b/src/emysql_saver.erl index 4cc00718..806008ea 100644 --- a/src/emysql_saver.erl +++ b/src/emysql_saver.erl @@ -35,17 +35,17 @@ save(_ConnOrPool, _Table, undefined) -> ok; save(_ConnOrPool, _Table, [[], _]) -> ok; -save(_ConnOrPool, _Table, {error, Reason}) -> - {error, Reason}; save(ConnOrPool, Table, RecordInput) -> DefOpt = [{auto_id, true}, {batch_size, 1000}], save(ConnOrPool, Table, RecordInput, DefOpt). -save(_ConnOrPool, _Table, {error, Reason}, _Options) -> - {error, Reason}; -save(ConnOrPool, Table, [Records, Fields] = _RecordInput, Options) -> +save(ConnOrPool, Table, [Record0, Fields] = _RecordInput, Options) -> [FieldPK | _ ] = Fields, + Records = case Record0 of + V1 when is_list(V1) -> V1; + V2 -> [V2] + end, case build_sql(Table, Records, Fields, Options) of {insert, Sqls} -> X = lists:foldl( From 26be1b3860c205a7795b1f114f9560b40ca0d28b Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Thu, 8 May 2014 15:18:37 +0800 Subject: [PATCH 34/61] clean up macros --- include/emysql.hrl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/emysql.hrl b/include/emysql.hrl index 70e1f81c..38edd472 100644 --- a/include/emysql.hrl +++ b/include/emysql.hrl @@ -181,7 +181,7 @@ -define(SERVER_STATUS_METADATA_CHANGED, 1024). % Wrap the query result in erlang record --define(AS_REC(RecAtom), [Rec, record_info(fields, RecAtom)]). --define(AS_VAL, as_val). +-define(AS_REC(RecAtom), [RecAtom, record_info(fields, RecAtom)]). +-define(AS_VAL, as_val). % -define(INPUT(RecAtom, Records), [Records, record_info(fields, RecAtom)]). From 04e68483fde74da04e84e288ac654d4237b7a127 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Fri, 9 May 2014 13:25:44 +0800 Subject: [PATCH 35/61] fix SqlFields bug in save --- src/emysql_saver.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/emysql_saver.erl b/src/emysql_saver.erl index 806008ea..11fa8fc0 100644 --- a/src/emysql_saver.erl +++ b/src/emysql_saver.erl @@ -172,7 +172,7 @@ generate_insert_sql(Table, UpdateFields, UpdateFIndex, Records, Options) -> _ -> "INSERT INTO " ++ type_utils:any_to_list(Table) end, - SqlFields = string:join(UpdateFields, ","), + SqlFields = string:join(lists:reverse(UpdateFields), ","), ValuesInSql = "(" ++ string:join(lists:duplicate(length(UpdateFields), "?"), ",") ++ ")", BatchCount = length(Records) div BatchSize, @@ -199,7 +199,7 @@ generate_insert_sql(Table, UpdateFields, UpdateFIndex, Records, Options) -> lists:foldl( fun(Idx, AccIn2) -> Val = element(Idx, RecItem), - [Val | AccIn2] + [Val | AccIn2] % That's why reverse SqlFields end, AccIn, UpdateFIndex) end, [], NRecords) end), From 8462979c208013be85adf073aa63b9b7ca6ad2f3 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Fri, 9 May 2014 15:36:07 +0800 Subject: [PATCH 36/61] fix return value of find_first --- src/emysql_query.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/emysql_query.erl b/src/emysql_query.erl index 2728a58c..51fe2714 100644 --- a/src/emysql_query.erl +++ b/src/emysql_query.erl @@ -92,7 +92,10 @@ find(ConnOrPool, Table, SqlOptions, [Rec, RecFields] = _AsRec) -> %%-------------------------------------------------------------------- find_first(ConnOrPool, Table, SqlOptions, AsRec) -> NSqlOptions = proplists:delete(limit, SqlOptions), - find(ConnOrPool, Table, [{limit, 1} | NSqlOptions], AsRec). + case find(ConnOrPool, Table, [{limit, 1} | NSqlOptions], AsRec) of + [ ] -> undefined; + [Val] -> Val + end. %%-------------------------------------------------------------------- From aed6bbaa97c10cb586efe2391de6c8c476a2ef17 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Fri, 9 May 2014 16:00:36 +0800 Subject: [PATCH 37/61] support created_at and updated_at --- src/emysql_saver.erl | 72 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 14 deletions(-) diff --git a/src/emysql_saver.erl b/src/emysql_saver.erl index 11fa8fc0..1338e5a7 100644 --- a/src/emysql_saver.erl +++ b/src/emysql_saver.erl @@ -11,6 +11,7 @@ %% API -export([save/3, save/4]). -export([find_or_create_by/4]). +-export([update_or_create_by/4]). -include("emysql.hrl"). %%%=================================================================== @@ -84,10 +85,11 @@ save(ConnOrPool, Table, [Record0, Fields] = _RecordInput, Options) -> %%-------------------------------------------------------------------- %% @doc %% -%% emysql_saver:find_or_create_by(pool, test, ["select * from test where data = ?", "hello"], -%% fun() -> -%% ?INPUT(#test{data = "find me"}) -%% end) +%% emysql_saver:find_or_create_by(pool, test, +%% ["select * from test where data = ?", "hello"], +%% fun() -> +%% ?INPUT(test, #test{data = "find me"}) +%% end) %% %% @spec %% @end @@ -95,10 +97,38 @@ save(ConnOrPool, Table, [Record0, Fields] = _RecordInput, Options) -> find_or_create_by(ConnOrPool, Table, FindSql, CreateFun) -> Result = emysql_query:find(ConnOrPool, FindSql), case CreateFun() of - [[Record], Fields] -> + [Record, Fields] -> case Result of #result_packet{rows = Rows} when length(Rows) =:= 0 -> - save(ConnOrPool, Table, [[Record], Fields]); + save(ConnOrPool, Table, [Record, Fields]); + #result_packet{} -> + emysql_conv:as_record(Result, element(1, Record), Fields); + Other -> + Other + end; + _ -> + io:format("Forget to wrap return value using ?INPUT?", []) + end. + +%%-------------------------------------------------------------------- +%% @doc +%% Options = [{ignore_null, true}] +%% emysql_saver:update_or_create_by(pool, test, +%% ["select * from test where data = ?", "hello"], +%% fun() -> +%% ?INPUT(test, #test{data = "find me"}) +%% end, Options) +%% +%% @spec +%% @end +%%-------------------------------------------------------------------- +update_or_create_by(ConnOrPool, Table, FindSql, Fun) -> + Result = emysql_query:find(ConnOrPool, FindSql), + case Fun() of + [Record, Fields] -> + case Result of + #result_packet{rows = Rows} when length(Rows) =:= 0 -> + save(ConnOrPool, Table, [Record, Fields]); #result_packet{} -> emysql_conv:as_record(Result, element(1, Record), Fields); Other -> @@ -117,20 +147,24 @@ find_or_create_by(ConnOrPool, Table, FindSql, CreateFun) -> build_sql(_Table, [] = _Records, _Fields, _Options) -> {error, no_input}; build_sql(Table, [Record | _Tail] = Records, Fields, Options) -> - {UpdateFields, UpdateFIndex, UpdateVals} = select_update_fields_index(Record, Fields, Options), case element(2, Record) of undefined -> % insert + {UpdateFields, UpdateFIndex, _UVals} = select_update_fields_index(insert, Record, + Fields, Options), + SqlAndValues = generate_insert_sql(Table, UpdateFields, UpdateFIndex, Records, Options), {insert, SqlAndValues}; - _ -> % update + _ -> % update + {UpdateFields, _FIndex, UpdateVals} = select_update_fields_index(update, Record, + Fields, Options), [PK | _] = Fields, PKPair = {type_utils:any_to_list(PK), element(2, Record)}, SqlAndValues = generate_update_sql(Table, UpdateFields, UpdateVals, PKPair), {update, SqlAndValues} end. -select_update_fields_index(Record, Fields, Options) -> +select_update_fields_index(InsertOrUpdate, Record, Fields, Options) -> {UAOptIsSet, UpdateAttrs} = case proplists:get_value(update_attrs, Options) of undefined -> {no, []}; @@ -145,20 +179,30 @@ select_update_fields_index(Record, Fields, Options) -> {_, UpdateFields, UpdateFIndex, UpdateVals} = lists:foldl( fun(Field, {Index, EffectedFields, EffectedIndex, Vals}) -> - case {Field, AutoId, UAOptIsSet, lists:member(Field, UpdateAttrs)} of - {id, true, _, _} -> + case {Field, AutoId, UAOptIsSet, InsertOrUpdate, lists:member(Field, UpdateAttrs)} of + {id, true, _, _, _} -> {Index + 1, EffectedFields, EffectedIndex, Vals}; - {_, _, no, _} -> + {created_at, _, _, insert, _} -> + NEffectedFields = [type_utils:any_to_list(Field) | EffectedFields], + NEffectedIndex = [(Index + 1) | EffectedIndex], + Val = datetime_utils:localtime_as_string(), + {Index + 1, NEffectedFields, NEffectedIndex, [Val | Vals]}; + {updated_at, _, _, _, _} -> + NEffectedFields = [type_utils:any_to_list(Field) | EffectedFields], + NEffectedIndex = [(Index + 1) | EffectedIndex], + Val = datetime_utils:localtime_as_string(), + {Index + 1, NEffectedFields, NEffectedIndex, [Val | Vals]}; + {_, _, no, _, _} -> NEffectedFields = [type_utils:any_to_list(Field) | EffectedFields], NEffectedIndex = [(Index + 1) | EffectedIndex], Val = element(Index + 1, Record), {Index + 1, NEffectedFields, NEffectedIndex, [Val | Vals]}; - {_, _, yes, true} -> + {_, _, yes, _, true} -> NEffectedFields = [type_utils:any_to_list(Field) | EffectedFields], NEffectedIndex = [(Index + 1) | EffectedIndex], Val = element(Index + 1, Record), {Index + 1, NEffectedFields, NEffectedIndex, [Val | Vals]}; - {_, _, yes, false} -> + {_, _, yes, _, false} -> {Index + 1, EffectedFields, EffectedIndex, Vals} end end, {1, [], [], []}, Fields), From 056a5d24bbf9a832c05ea49bcbf58300dc8c56f6 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Fri, 9 May 2014 17:52:06 +0800 Subject: [PATCH 38/61] add update_or_create_by --- src/emysql_saver.erl | 41 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/src/emysql_saver.erl b/src/emysql_saver.erl index 1338e5a7..264332d6 100644 --- a/src/emysql_saver.erl +++ b/src/emysql_saver.erl @@ -11,7 +11,8 @@ %% API -export([save/3, save/4]). -export([find_or_create_by/4]). --export([update_or_create_by/4]). +-export([update_or_create_by/4, + update_or_create_by/5]). -include("emysql.hrl"). %%%=================================================================== @@ -123,6 +124,9 @@ find_or_create_by(ConnOrPool, Table, FindSql, CreateFun) -> %% @end %%-------------------------------------------------------------------- update_or_create_by(ConnOrPool, Table, FindSql, Fun) -> + update_or_create_by(ConnOrPool, Table, FindSql, Fun, []). + +update_or_create_by(ConnOrPool, Table, FindSql, Fun, Options) -> Result = emysql_query:find(ConnOrPool, FindSql), case Fun() of [Record, Fields] -> @@ -130,7 +134,13 @@ update_or_create_by(ConnOrPool, Table, FindSql, Fun) -> #result_packet{rows = Rows} when length(Rows) =:= 0 -> save(ConnOrPool, Table, [Record, Fields]); #result_packet{} -> - emysql_conv:as_record(Result, element(1, Record), Fields); + RecInDb = emysql_conv:as_record(Result, element(1, Record), Fields), + case merge_rec(RecInDb, Record, Fields, Options) of + {changed, MergedRec} -> + save(ConnOrPool, Table, [MergedRec, Fields]); + _ -> + RecInDb + end; Other -> Other end; @@ -146,8 +156,8 @@ update_or_create_by(ConnOrPool, Table, FindSql, Fun) -> %% update table set f1 = ? build_sql(_Table, [] = _Records, _Fields, _Options) -> {error, no_input}; + build_sql(Table, [Record | _Tail] = Records, Fields, Options) -> - case element(2, Record) of undefined -> % insert {UpdateFields, UpdateFIndex, _UVals} = select_update_fields_index(insert, Record, @@ -265,3 +275,28 @@ generate_update_sql(Table, UpdateFields, UpdateVals, {FieldPK, PKVal}) -> Sql = string:join([SqlHead, "SET ", SqlSet, SqlTail], " "), {Sql, lists:append(UpdateVals, [PKVal])}. + + +merge_rec(Rec1, Rec2, Fields, Options) -> + IgnoreNil = proplists:get_value(ignore_nil, Options, false), + [_, RecChanged, MergedRec] = + lists:foldl(fun(Index, [Index, Changed, RecAcc]) -> + Val1 = element(Index + 1, Rec1), + Val2 = element(Index + 1, Rec2), + case {Changed, IgnoreNil, Val1 =:= Val2, Val2} of + {false, true, false, undefined} -> + [Index + 1, true, RecAcc]; + {false, true, false, _} -> + [Index + 1, true, setelement(Index + 1, RecAcc, Val2)]; + {false, _, true, _} -> + [Index + 1, false, RecAcc]; + {true, true, true, _} -> + [Index + 1, true, RecAcc]; + {_, false, false, _} -> + [Index + 1, true, setelement(Index + 1, RecAcc, Val2)] + end + end, [1, false, Rec1], Fields), + case RecChanged of + true -> {changed, MergedRec}; + _ -> {unchanged, MergedRec} + end. From 4cf46f7adcabff079feeaf88cf39bf8b3b687071 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Fri, 9 May 2014 18:19:20 +0800 Subject: [PATCH 39/61] fix merge_rec bug --- src/emysql_saver.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emysql_saver.erl b/src/emysql_saver.erl index 264332d6..cedd8329 100644 --- a/src/emysql_saver.erl +++ b/src/emysql_saver.erl @@ -280,7 +280,7 @@ generate_update_sql(Table, UpdateFields, UpdateVals, {FieldPK, PKVal}) -> merge_rec(Rec1, Rec2, Fields, Options) -> IgnoreNil = proplists:get_value(ignore_nil, Options, false), [_, RecChanged, MergedRec] = - lists:foldl(fun(Index, [Index, Changed, RecAcc]) -> + lists:foldl(fun(_F, [Index, Changed, RecAcc]) -> Val1 = element(Index + 1, Rec1), Val2 = element(Index + 1, Rec2), case {Changed, IgnoreNil, Val1 =:= Val2, Val2} of From 146d803a87908dab819ea9d7d87b749d1dda72e1 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Fri, 9 May 2014 18:44:32 +0800 Subject: [PATCH 40/61] fix bugs in update_or_create_by --- src/emysql_saver.erl | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/emysql_saver.erl b/src/emysql_saver.erl index cedd8329..312ad72f 100644 --- a/src/emysql_saver.erl +++ b/src/emysql_saver.erl @@ -134,12 +134,18 @@ update_or_create_by(ConnOrPool, Table, FindSql, Fun, Options) -> #result_packet{rows = Rows} when length(Rows) =:= 0 -> save(ConnOrPool, Table, [Record, Fields]); #result_packet{} -> - RecInDb = emysql_conv:as_record(Result, element(1, Record), Fields), - case merge_rec(RecInDb, Record, Fields, Options) of - {changed, MergedRec} -> - save(ConnOrPool, Table, [MergedRec, Fields]); - _ -> - RecInDb + case emysql_conv:as_record(Result, element(1, Record), Fields) of + [RecInDb] -> + case merge_rec(RecInDb, Record, Fields, Options) of + {changed, MergedRec} -> + save(ConnOrPool, Table, [MergedRec, Fields]); + _ -> + RecInDb + end; + L -> + io:format("Only one record expectedh here, but now there are ~p", + [length(L)]), + throw({error, many_records}) end; Other -> Other @@ -293,7 +299,9 @@ merge_rec(Rec1, Rec2, Fields, Options) -> {true, true, true, _} -> [Index + 1, true, RecAcc]; {_, false, false, _} -> - [Index + 1, true, setelement(Index + 1, RecAcc, Val2)] + [Index + 1, true, setelement(Index + 1, RecAcc, Val2)]; + {true, false, true, _} -> + [Index + 1, true, RecAcc] end end, [1, false, Rec1], Fields), case RecChanged of From 93719ca468157e82cf301983e3be46932ec8d705 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Mon, 12 May 2014 00:39:14 +0800 Subject: [PATCH 41/61] fix bug in update_or_create_by --- src/emysql_saver.erl | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/emysql_saver.erl b/src/emysql_saver.erl index 312ad72f..53177626 100644 --- a/src/emysql_saver.erl +++ b/src/emysql_saver.erl @@ -286,21 +286,23 @@ generate_update_sql(Table, UpdateFields, UpdateVals, {FieldPK, PKVal}) -> merge_rec(Rec1, Rec2, Fields, Options) -> IgnoreNil = proplists:get_value(ignore_nil, Options, false), [_, RecChanged, MergedRec] = - lists:foldl(fun(_F, [Index, Changed, RecAcc]) -> + lists:foldl(fun(Field, [Index, Changed, RecAcc]) -> Val1 = element(Index + 1, Rec1), Val2 = element(Index + 1, Rec2), - case {Changed, IgnoreNil, Val1 =:= Val2, Val2} of - {false, true, false, undefined} -> + case {Field, Changed, IgnoreNil, Val1 =:= Val2, Val2} of + {id, _, _, _, _} -> + [Index + 1, Changed, RecAcc]; + {_, false, true, false, undefined} -> [Index + 1, true, RecAcc]; - {false, true, false, _} -> + {_, false, true, false, _} -> [Index + 1, true, setelement(Index + 1, RecAcc, Val2)]; - {false, _, true, _} -> + {_, false, _, true, _} -> [Index + 1, false, RecAcc]; - {true, true, true, _} -> + {_, true, true, true, _} -> [Index + 1, true, RecAcc]; - {_, false, false, _} -> + {_, _, false, false, _} -> [Index + 1, true, setelement(Index + 1, RecAcc, Val2)]; - {true, false, true, _} -> + {_, true, false, true, _} -> [Index + 1, true, RecAcc] end end, [1, false, Rec1], Fields), From 21538da352eee25cae8f03579aac536790253575 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Mon, 12 May 2014 01:13:48 +0800 Subject: [PATCH 42/61] fix created_at/updated_at bug --- src/emysql_saver.erl | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/emysql_saver.erl b/src/emysql_saver.erl index 53177626..a0b905ec 100644 --- a/src/emysql_saver.erl +++ b/src/emysql_saver.erl @@ -200,12 +200,12 @@ select_update_fields_index(InsertOrUpdate, Record, Fields, Options) -> {Index + 1, EffectedFields, EffectedIndex, Vals}; {created_at, _, _, insert, _} -> NEffectedFields = [type_utils:any_to_list(Field) | EffectedFields], - NEffectedIndex = [(Index + 1) | EffectedIndex], + NEffectedIndex = [ timestamp | EffectedIndex], Val = datetime_utils:localtime_as_string(), {Index + 1, NEffectedFields, NEffectedIndex, [Val | Vals]}; {updated_at, _, _, _, _} -> NEffectedFields = [type_utils:any_to_list(Field) | EffectedFields], - NEffectedIndex = [(Index + 1) | EffectedIndex], + NEffectedIndex = [ timestamp | EffectedIndex], Val = datetime_utils:localtime_as_string(), {Index + 1, NEffectedFields, NEffectedIndex, [Val | Vals]}; {_, _, no, _, _} -> @@ -258,7 +258,11 @@ generate_insert_sql(Table, UpdateFields, UpdateFIndex, Records, Options) -> fun(RecItem, AccIn) -> lists:foldl( fun(Idx, AccIn2) -> - Val = element(Idx, RecItem), + Val = case Idx of + timestamp -> + datetime_utils:localtime_as_string(); + _ -> element(Idx, RecItem) + end, [Val | AccIn2] % That's why reverse SqlFields end, AccIn, UpdateFIndex) end, [], NRecords) From e461b1595a7336e57633b8978b660e056f54910f Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Tue, 13 May 2014 18:01:42 +0800 Subject: [PATCH 43/61] ignore created_at in updating records --- src/emysql_saver.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/emysql_saver.erl b/src/emysql_saver.erl index a0b905ec..df83bd2c 100644 --- a/src/emysql_saver.erl +++ b/src/emysql_saver.erl @@ -203,6 +203,8 @@ select_update_fields_index(InsertOrUpdate, Record, Fields, Options) -> NEffectedIndex = [ timestamp | EffectedIndex], Val = datetime_utils:localtime_as_string(), {Index + 1, NEffectedFields, NEffectedIndex, [Val | Vals]}; + {created_at, _, _, update, _} -> + {Index + 1, EffectedFields, EffectedIndex, Vals}; {updated_at, _, _, _, _} -> NEffectedFields = [type_utils:any_to_list(Field) | EffectedFields], NEffectedIndex = [ timestamp | EffectedIndex], From 74e61ffa809f8aa4ed735292ebb18c69edc93042 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Tue, 13 May 2014 18:20:45 +0800 Subject: [PATCH 44/61] return updated rec when saving one rec --- src/emysql_saver.erl | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/emysql_saver.erl b/src/emysql_saver.erl index df83bd2c..1ee7cb75 100644 --- a/src/emysql_saver.erl +++ b/src/emysql_saver.erl @@ -75,12 +75,23 @@ save(ConnOrPool, Table, [Record0, Fields] = _RecordInput, Options) -> [V] -> V end; {update, {Sql, Values} } -> - case ConnOrPool of - #emysql_connection{} = Conn -> - emysql_conn:execute(Conn, Sql, Values); - Pool -> - emysql:execute(Pool, Sql, Values) - end + Result = case ConnOrPool of + #emysql_connection{} = Conn -> + emysql_conn:execute(Conn, Sql, Values); + Pool -> + emysql:execute(Pool, Sql, Values) + end, + case Result of + #ok_packet{affected_rows = Affected} -> + case {Affected, length(Records)} of + {1, 1} -> + [Record] = Records, + Record; + _ -> Records + end; + #error_packet{code = Code, msg = Msg} -> + throw({Code, Msg}) + end end. %%-------------------------------------------------------------------- From 8dd8960ae71410b1254a0673c09eeb95001974a2 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Tue, 10 Jun 2014 14:51:15 +0800 Subject: [PATCH 45/61] add AccIn to collect find_each result --- src/emysql_query.erl | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/src/emysql_query.erl b/src/emysql_query.erl index 51fe2714..c14b23f6 100644 --- a/src/emysql_query.erl +++ b/src/emysql_query.erl @@ -11,7 +11,7 @@ %% API -export([find/2, find/3, find/4]). -export([find_first/4]). --export([find_each/5, find_each/6]). +-export([find_each/4, find_each/5, find_each/6]). -include("emysql.hrl"). @@ -113,10 +113,16 @@ find_first(ConnOrPool, Table, SqlOptions, AsRec) -> %% @spec %% @end %%-------------------------------------------------------------------- -find_each(ConnOrPool, Table, SqlOptions, AsRec, Fun) -> - find_each(ConnOrPool, Table, SqlOptions, 1000, AsRec, Fun). +find_each(ConnOrPool, Table, AsRec, Fun) -> + find_each(ConnOrPool, Table, [], 1000, AsRec, Fun, undefined). -find_each(ConnOrPool, Table, SqlOptions, BatchSize, AsRec, Fun) -> +find_each(ConnOrPool, Table, AsRec, Fun, AccIn) -> + find_each(ConnOrPool, Table, [], 1000, AsRec, Fun, AccIn). + +find_each(ConnOrPool, Table, SqlOptions, AsRec, Fun, AccIn) -> + find_each(ConnOrPool, Table, SqlOptions, 1000, AsRec, Fun, AccIn). + +find_each(ConnOrPool, Table, SqlOptions, BatchSize, AsRec, Fun, AccIn) -> BaseId = 0, {[FindSql, FindCondVals], @@ -137,7 +143,7 @@ find_each(ConnOrPool, Table, SqlOptions, BatchSize, AsRec, Fun) -> {_, false} -> Total end, do_find_each(ConnOrPool, Table, FindSql, FindCondVals, BatchSize, - AsRec, Fun, Remain, BaseId); + AsRec, Fun, AccIn, Remain, BaseId); #error_packet{code = Code, msg = Msg} -> throw({Code, Msg}); _ -> @@ -223,7 +229,8 @@ build_sql(Table, SqlOptions, BatchSize, BaseId) -> -do_find_each(ConnOrPool, Table, Sql, CondVals, BatchSize, [Rec, RecFields] = AsRec, Fun, Remain, BaseId) -> +do_find_each(ConnOrPool, Table, Sql, CondVals, BatchSize, [Rec, RecFields] = AsRec, + Fun, AccIn, Remain, BaseId) -> NCondVals = lists:append(lists_utils:droplast(CondVals), [BaseId]), Result = case ConnOrPool of @@ -234,18 +241,22 @@ do_find_each(ConnOrPool, Table, Sql, CondVals, BatchSize, [Rec, RecFields] = AsR end, case Result of #result_packet{rows = Rows} -> - emysql_conv:as_record(Result, Rec, RecFields, Fun), + Rs = emysql_conv:as_record(Result, Rec, RecFields, Fun), + NAccIn = case AccIn of + L when is_list(L) -> lists:append(AccIn, Rs); + _ -> undefined + end, LastRow = lists_utils:last(Rows), [NextId | _Tail] = LastRow, case Remain - BatchSize > 0 of true -> do_find_each(ConnOrPool, Table, Sql, CondVals, BatchSize, AsRec, - Fun, (Remain - BatchSize), NextId); + Fun, NAccIn, (Remain - BatchSize), NextId); false -> - ok + NAccIn end; _ -> - failed + AccIn end. From 3c04fa4bc9a20f2a57a1c452af4c93fc5e1242ea Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Fri, 13 Jun 2014 17:01:47 +0800 Subject: [PATCH 46/61] fix ?AS_VAL bug --- src/emysql_query.erl | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/emysql_query.erl b/src/emysql_query.erl index c14b23f6..e90b612d 100644 --- a/src/emysql_query.erl +++ b/src/emysql_query.erl @@ -72,8 +72,12 @@ find(ConnOrPool, Table, SqlOptions) -> find(ConnOrPool, Table, SqlOptions, ?AS_VAL) -> case find(ConnOrPool, Table, SqlOptions) of #result_packet{rows = [[R] ]} -> R; - #result_packet{rows = [ Rs ]} -> Rs; - Other -> Other + #result_packet{rows = Rs } -> + lists:foldl(fun([R], AccIn) -> + lists:append(AccIn, [R]) + end, [], Rs); + #error_packet{code = Code, msg = Msg} -> + throw({Code, Msg}) end; find(ConnOrPool, Table, SqlOptions, [Rec, RecFields] = _AsRec) -> @@ -90,7 +94,11 @@ find(ConnOrPool, Table, SqlOptions, [Rec, RecFields] = _AsRec) -> %% @spec %% @end %%-------------------------------------------------------------------- -find_first(ConnOrPool, Table, SqlOptions, AsRec) -> +find_first(ConnOrPool, Table, SqlOptions, ?AS_VAL) -> + NSqlOptions = proplists:delete(limit, SqlOptions), + find(ConnOrPool, Table, [{limit, 1} | NSqlOptions], ?AS_VAL); + +find_first(ConnOrPool, Table, SqlOptions, [_Rec, _RecFields] = AsRec) -> NSqlOptions = proplists:delete(limit, SqlOptions), case find(ConnOrPool, Table, [{limit, 1} | NSqlOptions], AsRec) of [ ] -> undefined; From 1704d77416583a7ad6b005f0ad9e67fe3b205f5e Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Mon, 16 Jun 2014 16:54:41 +0800 Subject: [PATCH 47/61] fix find_each bug --- src/emysql_query.erl | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/src/emysql_query.erl b/src/emysql_query.erl index e90b612d..e3a85399 100644 --- a/src/emysql_query.erl +++ b/src/emysql_query.erl @@ -40,13 +40,13 @@ find(ConnOrPool, [RawSql | Values]) -> %%-------------------------------------------------------------------- %% @doc %% e.g: find(my_pool, users, [ {select, ["name", "age"]}, -%% {where, ["age > ?", Age]}, +%% {where, ["age > ? AND color in (?)", Age, "blue, red"]}, %% {order, "id desc"}, %% {limit, 100 }], ?AS_REC(user)) %% or %% %% find(my_pool, users, [ {select, ["name", "age"]}, -%% {where, [{age, '>', Age}]}, +%% {where, [{age, '>', Age}, {color, in, ["blue", "red"]} ]}, %% {order, "id desc"}, %% {limit, 100 }], ?AS_REC(user)) %% @@ -185,11 +185,15 @@ build_sql(Table, SqlOptions, BatchSize, BaseId) -> {K, between, [V1, V2]} -> S = type_utils:any_to_list(K) ++ " BETWEEN ? AND ? ", {[S | StmtAcc], [V1, V2 | ValAcc]}; - {K, OP, V3} -> + {K, in, V3} when is_list(V3) -> + S = type_utils:any_to_list(K) ++ " IN (?) ", + NV3 = string:join(V3, ", "), + {[S | StmtAcc], [NV3 | ValAcc]}; + {K, OP, V4} -> S = string:join([type_utils:any_to_list(K), type_utils:any_to_list(OP), "?"], " "), - {[S | StmtAcc], [V3 | ValAcc] }; + {[S | StmtAcc], [V4 | ValAcc] }; {K, V4} -> S = type_utils:any_to_list(K) ++ " = ?", {[S | StmtAcc], [V4 | ValAcc]}; @@ -250,18 +254,22 @@ do_find_each(ConnOrPool, Table, Sql, CondVals, BatchSize, [Rec, RecFields] = AsR case Result of #result_packet{rows = Rows} -> Rs = emysql_conv:as_record(Result, Rec, RecFields, Fun), - NAccIn = case AccIn of - L when is_list(L) -> lists:append(AccIn, Rs); - _ -> undefined - end, - LastRow = lists_utils:last(Rows), - [NextId | _Tail] = LastRow, - case Remain - BatchSize > 0 of - true -> - do_find_each(ConnOrPool, Table, Sql, CondVals, BatchSize, AsRec, - Fun, NAccIn, (Remain - BatchSize), NextId); - false -> - NAccIn + case Rs of + [] -> AccIn; + _ -> + NAccIn = case AccIn of + L when is_list(L) -> lists:append(AccIn, Rs); + _ -> undefined + end, + LastRow = lists_utils:last(Rows), + [NextId | _Tail] = LastRow, + case Remain - BatchSize > 0 of + true -> + do_find_each(ConnOrPool, Table, Sql, CondVals, BatchSize, AsRec, + Fun, NAccIn, (Remain - BatchSize), NextId); + false -> + NAccIn + end end; _ -> AccIn From 2a9bfb525ef554d9b0730614e981f4ee2f007779 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Mon, 16 Jun 2014 17:04:15 +0800 Subject: [PATCH 48/61] find_or_create_by returns one object --- src/emysql_saver.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/emysql_saver.erl b/src/emysql_saver.erl index 1ee7cb75..abfd5462 100644 --- a/src/emysql_saver.erl +++ b/src/emysql_saver.erl @@ -114,7 +114,8 @@ find_or_create_by(ConnOrPool, Table, FindSql, CreateFun) -> #result_packet{rows = Rows} when length(Rows) =:= 0 -> save(ConnOrPool, Table, [Record, Fields]); #result_packet{} -> - emysql_conv:as_record(Result, element(1, Record), Fields); + [R | _] = emysql_conv:as_record(Result, element(1, Record), Fields), + R; Other -> Other end; From b70157fb06549d190298df482894727aee678b00 Mon Sep 17 00:00:00 2001 From: zhangnan Date: Thu, 25 Dec 2014 17:54:33 +0800 Subject: [PATCH 49/61] fix generate_insert_sql bug --- src/emysql_saver.erl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/emysql_saver.erl b/src/emysql_saver.erl index abfd5462..160c90f9 100644 --- a/src/emysql_saver.erl +++ b/src/emysql_saver.erl @@ -282,13 +282,15 @@ generate_insert_sql(Table, UpdateFields, UpdateFIndex, Records, Options) -> end, [], NRecords) end), case BatchRemainSize of - 0 -> [{Batch1Sql, lists:merge(RecBatchValues)}]; + 0 -> + [{Batch1Sql, FRecBatchValues} || FRecBatchValues <- lists:delete([], RecBatchValues)]; _ -> {Batch1Values, Batch2Values} = lists:split(BatchCount, RecBatchValues), case Batch1Sql of undefined -> [{Batch2Sql, lists:merge(Batch2Values)}]; - _ -> [{Batch1Sql, lists:merge(Batch1Values)}, - {Batch2Sql, lists:merge(Batch2Values)}] + _ -> + lists:merge([{Batch1Sql, FBatch1Values} || FBatch1Values <- Batch1Values], + [{Batch2Sql, lists:merge(Batch2Values)}]) end end. From 1db6cf58f0bc7b72c6a21006386b2960bb971ca8 Mon Sep 17 00:00:00 2001 From: zhangnan Date: Mon, 2 Feb 2015 15:18:26 +0800 Subject: [PATCH 50/61] add fold function to emysql_query module --- src/emysql_conv.erl | 30 +++++++++++++++++++++------- src/emysql_query.erl | 47 +++++++++++++++++++++++++++++++++----------- 2 files changed, 58 insertions(+), 19 deletions(-) diff --git a/src/emysql_conv.erl b/src/emysql_conv.erl index 28af9dd8..33b8a915 100644 --- a/src/emysql_conv.erl +++ b/src/emysql_conv.erl @@ -11,7 +11,8 @@ as_json/1, as_proplist/1, as_record/3, - as_record/4 + as_record/4, + as_record/5 ]). %% @see emysql:as_dict/1 @@ -36,7 +37,7 @@ as_proplist(Res = #result_packet{field_list=Cols,rows=Rows}) when is_list(Cols), end || R <- Rows]. %% @see emysql:as_record/1 -as_record(Result = #result_packet{}, RecordName, Fields, Fun) when is_atom(RecordName), is_list(Fields), is_function(Fun) -> +as_record(Result = #result_packet{}, RecordName, Fields, Fun, FunState) when is_atom(RecordName), is_list(Fields), is_function(Fun) -> Columns = Result#result_packet.field_list, S = lists:seq(1, length(Columns)), @@ -50,11 +51,26 @@ as_record(Result = #result_packet{}, RecordName, Fields, Fun) when is_atom(Recor end end, Fs = [ F(FieldName) || FieldName <- Fields ], - F1 = fun(Row) -> - RecordData = [ Fx(Row) || Fx <- Fs ], - Fun(list_to_tuple([RecordName|RecordData])) - end, - [ F1(Row) || Row <- Result#result_packet.rows ]. + + case FunState of + undefined -> + lists:foldl( + fun(Row, RAcc) -> + RecordData = [ Fx(Row) || Fx <- Fs ], + R = Fun(list_to_tuple([RecordName|RecordData])), + [R | RAcc] + end, [], Result#result_packet.rows); + _ -> + lists:foldl( + fun(Row, {RAcc, FAcc}) -> + RecordData = [ Fx(Row) || Fx <- Fs ], + {R, NFAcc} = Fun(list_to_tuple([RecordName|RecordData]), FAcc), + {[R | RAcc], NFAcc} + end, {[], FunState}, Result#result_packet.rows) + end. + +as_record(Result = #result_packet{}, RecordName, Fields, Fun) when is_atom(RecordName), is_list(Fields), is_function(Fun) -> + as_record(Result, RecordName, Fields, fun(A) -> A end, undefined). as_record(Result = #result_packet{}, RecordName, Fields) when is_atom(RecordName), is_list(Fields) -> as_record(Result, RecordName, Fields, fun(A) -> A end). diff --git a/src/emysql_query.erl b/src/emysql_query.erl index e3a85399..aa608871 100644 --- a/src/emysql_query.erl +++ b/src/emysql_query.erl @@ -12,7 +12,7 @@ -export([find/2, find/3, find/4]). -export([find_first/4]). -export([find_each/4, find_each/5, find_each/6]). - +-export([foldl/5, foldl/6, foldl/7]). -include("emysql.hrl"). @@ -105,6 +105,29 @@ find_first(ConnOrPool, Table, SqlOptions, [_Rec, _RecFields] = AsRec) -> [Val] -> Val end. +%%-------------------------------------------------------------------- +%% @doc +%% +%% e.g: +%% Fun = fun(#user{} = U, FunState) -> +%% {R, NFunState} = do_something_to(U, FunState) +%% end, +%% find(my_pool, users, [ {select, ["name", "age"]}, +%% {where, ["age > ?", Age]}, +%% {order, "id desc"}, +%% {limit, 100 } ], 10, ?AS_REC(user), Fun, FunState) +%% +%% @spec +%% @end +%%-------------------------------------------------------------------- +foldl(ConnOrPool, Table, AsRec, Fun, FunState) -> + find_each(ConnOrPool, Table, [], 1000, AsRec, Fun, FunState, undefined). + +foldl(ConnOrPool, Table, AsRec, Fun, FunState, AccIn) -> + find_each(ConnOrPool, Table, [], 1000, AsRec, Fun, FunState, AccIn). + +foldl(ConnOrPool, Table, SqlOptions, AsRec, Fun, FunState, AccIn) -> + find_each(ConnOrPool, Table, SqlOptions, 1000, AsRec, Fun, FunState, AccIn). %%-------------------------------------------------------------------- %% @doc @@ -122,20 +145,20 @@ find_first(ConnOrPool, Table, SqlOptions, [_Rec, _RecFields] = AsRec) -> %% @end %%-------------------------------------------------------------------- find_each(ConnOrPool, Table, AsRec, Fun) -> - find_each(ConnOrPool, Table, [], 1000, AsRec, Fun, undefined). + find_each(ConnOrPool, Table, [], 1000, AsRec, Fun, undefined, undefined). find_each(ConnOrPool, Table, AsRec, Fun, AccIn) -> - find_each(ConnOrPool, Table, [], 1000, AsRec, Fun, AccIn). + find_each(ConnOrPool, Table, [], 1000, AsRec, Fun, undefined, AccIn). find_each(ConnOrPool, Table, SqlOptions, AsRec, Fun, AccIn) -> - find_each(ConnOrPool, Table, SqlOptions, 1000, AsRec, Fun, AccIn). + find_each(ConnOrPool, Table, SqlOptions, 1000, AsRec, Fun, undefined, AccIn). -find_each(ConnOrPool, Table, SqlOptions, BatchSize, AsRec, Fun, AccIn) -> +find_each(ConnOrPool, Table, SqlOptions, BatchSize, AsRec, Fun, FunState, AccIn) -> BaseId = 0, - + {[FindSql, FindCondVals], [CountSql, CountCondVals]} = build_sql(Table, SqlOptions, BatchSize, BaseId), - + Result = case ConnOrPool of #emysql_connection{} = Conn -> emysql_conn:execute(Conn, CountSql, CountCondVals); @@ -151,7 +174,7 @@ find_each(ConnOrPool, Table, SqlOptions, BatchSize, AsRec, Fun, AccIn) -> {_, false} -> Total end, do_find_each(ConnOrPool, Table, FindSql, FindCondVals, BatchSize, - AsRec, Fun, AccIn, Remain, BaseId); + AsRec, Fun, FunState, AccIn, Remain, BaseId); #error_packet{code = Code, msg = Msg} -> throw({Code, Msg}); _ -> @@ -242,7 +265,7 @@ build_sql(Table, SqlOptions, BatchSize, BaseId) -> do_find_each(ConnOrPool, Table, Sql, CondVals, BatchSize, [Rec, RecFields] = AsRec, - Fun, AccIn, Remain, BaseId) -> + Fun, FunState, AccIn, Remain, BaseId) -> NCondVals = lists:append(lists_utils:droplast(CondVals), [BaseId]), Result = case ConnOrPool of @@ -253,7 +276,7 @@ do_find_each(ConnOrPool, Table, Sql, CondVals, BatchSize, [Rec, RecFields] = AsR end, case Result of #result_packet{rows = Rows} -> - Rs = emysql_conv:as_record(Result, Rec, RecFields, Fun), + Rs = emysql_conv:as_record(Result, Rec, RecFields, Fun, FunState), case Rs of [] -> AccIn; _ -> @@ -264,9 +287,9 @@ do_find_each(ConnOrPool, Table, Sql, CondVals, BatchSize, [Rec, RecFields] = AsR LastRow = lists_utils:last(Rows), [NextId | _Tail] = LastRow, case Remain - BatchSize > 0 of - true -> + true -> do_find_each(ConnOrPool, Table, Sql, CondVals, BatchSize, AsRec, - Fun, NAccIn, (Remain - BatchSize), NextId); + Fun, FunState, NAccIn, (Remain - BatchSize), NextId); false -> NAccIn end From bd89e7a95d9df263e7ca58fa96bf0e7443c5b0a5 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Tue, 10 Feb 2015 17:48:27 +0800 Subject: [PATCH 51/61] enhanced emysql_query:find_each --- src/emysql_conv.erl | 33 +++++------ src/emysql_query.erl | 138 ++++++++++++++++++++++--------------------- src/emysql_saver.erl | 18 +++++- 3 files changed, 103 insertions(+), 86 deletions(-) diff --git a/src/emysql_conv.erl b/src/emysql_conv.erl index 33b8a915..0ad2b33a 100644 --- a/src/emysql_conv.erl +++ b/src/emysql_conv.erl @@ -11,8 +11,7 @@ as_json/1, as_proplist/1, as_record/3, - as_record/4, - as_record/5 + as_record/4 ]). %% @see emysql:as_dict/1 @@ -37,7 +36,7 @@ as_proplist(Res = #result_packet{field_list=Cols,rows=Rows}) when is_list(Cols), end || R <- Rows]. %% @see emysql:as_record/1 -as_record(Result = #result_packet{}, RecordName, Fields, Fun, FunState) when is_atom(RecordName), is_list(Fields), is_function(Fun) -> +as_record(Result = #result_packet{}, RecordName, Fields, Fun, AccIn) when is_atom(RecordName), is_list(Fields), is_function(Fun) -> Columns = Result#result_packet.field_list, S = lists:seq(1, length(Columns)), @@ -52,25 +51,25 @@ as_record(Result = #result_packet{}, RecordName, Fields, Fun, FunState) when is_ end, Fs = [ F(FieldName) || FieldName <- Fields ], - case FunState of + case AccIn of undefined -> - lists:foldl( - fun(Row, RAcc) -> - RecordData = [ Fx(Row) || Fx <- Fs ], - R = Fun(list_to_tuple([RecordName|RecordData])), - [R | RAcc] - end, [], Result#result_packet.rows); + F1 = fun(Row) -> + RecordData = [ Fx(Row) || Fx <- Fs ], + Fun(list_to_tuple([RecordName|RecordData]), AccIn) + end, + [ F1(Row) || Row <- Result#result_packet.rows ]; _ -> - lists:foldl( - fun(Row, {RAcc, FAcc}) -> - RecordData = [ Fx(Row) || Fx <- Fs ], - {R, NFAcc} = Fun(list_to_tuple([RecordName|RecordData]), FAcc), - {[R | RAcc], NFAcc} - end, {[], FunState}, Result#result_packet.rows) + lists:foldl(fun(Row, {RsAccIn, FunAccIn}) -> + RecordData = [ Fx(Row) || Fx <- Fs ], + Rec = list_to_tuple([RecordName|RecordData]), + NFunAccIn = Fun(Rec, FunAccIn), + {[lists:append(Rec, RsAccIn), NFunAccIn]} + end, {[], AccIn}, Result#result_packet.rows) end. as_record(Result = #result_packet{}, RecordName, Fields, Fun) when is_atom(RecordName), is_list(Fields), is_function(Fun) -> - as_record(Result, RecordName, Fields, fun(A) -> A end, undefined). + as_record(Result = #result_packet{}, RecordName, Fields, Fun, undefined). + as_record(Result = #result_packet{}, RecordName, Fields) when is_atom(RecordName), is_list(Fields) -> as_record(Result, RecordName, Fields, fun(A) -> A end). diff --git a/src/emysql_query.erl b/src/emysql_query.erl index aa608871..91c9fd72 100644 --- a/src/emysql_query.erl +++ b/src/emysql_query.erl @@ -12,7 +12,6 @@ -export([find/2, find/3, find/4]). -export([find_first/4]). -export([find_each/4, find_each/5, find_each/6]). --export([foldl/5, foldl/6, foldl/7]). -include("emysql.hrl"). @@ -105,29 +104,6 @@ find_first(ConnOrPool, Table, SqlOptions, [_Rec, _RecFields] = AsRec) -> [Val] -> Val end. -%%-------------------------------------------------------------------- -%% @doc -%% -%% e.g: -%% Fun = fun(#user{} = U, FunState) -> -%% {R, NFunState} = do_something_to(U, FunState) -%% end, -%% find(my_pool, users, [ {select, ["name", "age"]}, -%% {where, ["age > ?", Age]}, -%% {order, "id desc"}, -%% {limit, 100 } ], 10, ?AS_REC(user), Fun, FunState) -%% -%% @spec -%% @end -%%-------------------------------------------------------------------- -foldl(ConnOrPool, Table, AsRec, Fun, FunState) -> - find_each(ConnOrPool, Table, [], 1000, AsRec, Fun, FunState, undefined). - -foldl(ConnOrPool, Table, AsRec, Fun, FunState, AccIn) -> - find_each(ConnOrPool, Table, [], 1000, AsRec, Fun, FunState, AccIn). - -foldl(ConnOrPool, Table, SqlOptions, AsRec, Fun, FunState, AccIn) -> - find_each(ConnOrPool, Table, SqlOptions, 1000, AsRec, Fun, FunState, AccIn). %%-------------------------------------------------------------------- %% @doc @@ -136,29 +112,33 @@ foldl(ConnOrPool, Table, SqlOptions, AsRec, Fun, FunState, AccIn) -> %% Fun = fun(#user{} = U) -> %% do_something_to(U) %% end, -%% find(my_pool, users, [ {select, ["name", "age"]}, -%% {where, ["age > ?", Age]}, -%% {order, "id desc"}, -%% {limit, 100 } ], 10, ?AS_REC(user), Fun) +%% find_each(my_pool, users, ?AS_REC(user), Fun). +%% +%% find_each(my_pool, users, [ {select, ["name", "age"]}, +%% {where, ["age > ?", Age]}, +%% % {order, "id desc"}, +%% {order, [id, "DESC"] }, +%% {limit, 100 } ], 10, ?AS_REC(user), Fun). %% %% @spec %% @end %%-------------------------------------------------------------------- find_each(ConnOrPool, Table, AsRec, Fun) -> - find_each(ConnOrPool, Table, [], 1000, AsRec, Fun, undefined, undefined). + find_each(ConnOrPool, Table, [], 1000, AsRec, Fun, undefined). find_each(ConnOrPool, Table, AsRec, Fun, AccIn) -> - find_each(ConnOrPool, Table, [], 1000, AsRec, Fun, undefined, AccIn). + find_each(ConnOrPool, Table, [], 1000, AsRec, Fun, AccIn). find_each(ConnOrPool, Table, SqlOptions, AsRec, Fun, AccIn) -> - find_each(ConnOrPool, Table, SqlOptions, 1000, AsRec, Fun, undefined, AccIn). - -find_each(ConnOrPool, Table, SqlOptions, BatchSize, AsRec, Fun, FunState, AccIn) -> - BaseId = 0, + find_each(ConnOrPool, Table, SqlOptions, 1000, AsRec, Fun, AccIn). +find_each(ConnOrPool, Table, SqlOptions, BatchSize, AsRec, Fun, AccIn) -> + BaseId = undefined, + {[FindSql, FindCondVals], - [CountSql, CountCondVals]} = build_sql(Table, SqlOptions, BatchSize, BaseId), - + [CountSql, CountCondVals], + [OField, OSort]} = build_sql(Table, SqlOptions, BatchSize, BaseId), + Result = case ConnOrPool of #emysql_connection{} = Conn -> emysql_conn:execute(Conn, CountSql, CountCondVals); @@ -173,8 +153,13 @@ find_each(ConnOrPool, Table, SqlOptions, BatchSize, AsRec, Fun, FunState, AccIn) {_, true} -> Limit; {_, false} -> Total end, - do_find_each(ConnOrPool, Table, FindSql, FindCondVals, BatchSize, - AsRec, Fun, FunState, AccIn, Remain, BaseId); + %BaseId = case OSort of + % "ASC" -> 0; + % "DESC" -> infinite + % end, + %do_find_each(ConnOrPool, Table, FindSql, FindCondVals, BatchSize, + % AsRec, Fun, AccIn, Remain, BaseId); + do_find_each(ConnOrPool, Table, SqlOptions, BatchSize, AsRec, Fun, AccIn, Remain, BaseId); #error_packet{code = Code, msg = Msg} -> throw({Code, Msg}); _ -> @@ -228,21 +213,29 @@ build_sql(Table, SqlOptions, BatchSize, BaseId) -> [ Cond ] -> {"WHERE " ++ Cond, [ ]}; [Cond | Values] -> {"WHERE " ++ Cond, Values} end, - {Where2, CondVals2} = - case {BaseId, Where} of - {undefined, _} -> {Where, CondVals}; - {_, ""} -> {"WHERE id > ?", [BaseId]}; - _ -> - {Where ++ "AND id > ?", lists:append(CondVals, [BaseId])} + + [Order, OField, OSort] = + case proplists:get_value(order, SqlOptions) of + undefined -> ["", id, "ASC"]; + [OrderField, OrderSort] -> + ["ORDER BY " ++ type_utils:any_to_list(OrderField) ++ " " ++ type_utils:any_to_list(OrderSort), + OrderField, OrderSort] end, - - Order = case proplists:get_value(order, SqlOptions) of - undefined -> ""; - OrderBy -> "ORDER BY " ++ type_utils:any_to_list(OrderBy) - end, - - + {Where2, CondVals2} = + case {BaseId, OSort, Where} of + {infinate, _, _} -> {Where, CondVals}; + {undefined, _, _} -> {Where, CondVals}; + {_, "ASC", ""} -> {"WHERE " ++ type_utils:any_to_list(OField) ++ " > ?", [BaseId]}; + {_, "DESC", ""} -> {"WHERE " ++ type_utils:any_to_list(OField) ++ " < ?", [BaseId]}; + {_, "ASC", _} -> + {Where ++ " AND " ++ type_utils:any_to_list(OField) ++ " > ?", + lists:append(CondVals, [BaseId])}; + {_, "DESC", _ } -> + {Where ++ " AND " ++ type_utils:any_to_list(OField) ++ " < ?", + lists:append(CondVals, [BaseId])} + end, + Limit = case BatchSize of undefined -> @@ -255,43 +248,52 @@ build_sql(Table, SqlOptions, BatchSize, BaseId) -> _ -> "LIMIT " ++ type_utils:any_to_list(BatchSize) end, - Table2 = type_utils:any_to_list(Table), FindSql = string:join(["SELECT", SelectFields, "FROM", Table2, Where2, Order, Limit], " "), CountSql = string:join(["SELECT COUNT(1) FROM", Table2, Where], " "), - {[FindSql, CondVals2], [CountSql, CondVals]}. + {[FindSql, CondVals2], [CountSql, CondVals], [OField, OSort]}. +do_find_each(ConnOrPool, Table, SqlOptions, BatchSize, [Rec, RecFields] = AsRec, + Fun, AccIn, Remain, BaseId) -> + {[Sql, CondVals], + [_CountSql, _CountCondVals], + [OrderField, _OSort]} = build_sql(Table, SqlOptions, BatchSize, BaseId), + do_find_each(ConnOrPool, Table, Sql, CondVals, BatchSize, AsRec, Fun, AccIn, Remain, OrderField, BaseId). do_find_each(ConnOrPool, Table, Sql, CondVals, BatchSize, [Rec, RecFields] = AsRec, - Fun, FunState, AccIn, Remain, BaseId) -> - NCondVals = lists:append(lists_utils:droplast(CondVals), [BaseId]), + Fun, AccIn, Remain, OrderField, BaseId) -> Result = case ConnOrPool of #emysql_connection{} = Conn -> - emysql_conn:execute(Conn, Sql, NCondVals); + emysql_conn:execute(Conn, Sql, CondVals); Pool -> - emysql:execute(Pool, Sql, NCondVals) + emysql:execute(Pool, Sql, CondVals) end, case Result of #result_packet{rows = Rows} -> - Rs = emysql_conv:as_record(Result, Rec, RecFields, Fun, FunState), - case Rs of - [] -> AccIn; - _ -> - NAccIn = case AccIn of - L when is_list(L) -> lists:append(AccIn, Rs); - _ -> undefined - end, - LastRow = lists_utils:last(Rows), - [NextId | _Tail] = LastRow, + case emysql_conv:as_record(Result, Rec, RecFields, Fun, AccIn) of + {[], NAccIn} -> NAccIn; + {Rs, NAccIn} -> + LastRow = lists_utils:last(Rs), + NextId = tuple_utils:value(OrderField, LastRow, RecFields), case Remain - BatchSize > 0 of true -> do_find_each(ConnOrPool, Table, Sql, CondVals, BatchSize, AsRec, - Fun, FunState, NAccIn, (Remain - BatchSize), NextId); + Fun, NAccIn, (Remain - BatchSize), OrderField, NextId); false -> NAccIn + end; + Rs when is_list(Rs) -> + LastRow = lists_utils:last(Rs), + NextId = tuple_utils:value(OrderField, LastRow, RecFields), + case Remain - BatchSize > 0 of + true -> + do_find_each(ConnOrPool, Table, Sql, CondVals, BatchSize, AsRec, + Fun, undefined, (Remain - BatchSize), OrderField, NextId); + false -> + ok end end; _ -> diff --git a/src/emysql_saver.erl b/src/emysql_saver.erl index 160c90f9..3f07bf49 100644 --- a/src/emysql_saver.erl +++ b/src/emysql_saver.erl @@ -28,6 +28,9 @@ %% %% or %% +%% +%% or +%% %% emysql_saver:save(pool, test, ?INPUT(#test{data = "hello"})). %% %% @spec @@ -281,9 +284,21 @@ generate_insert_sql(Table, UpdateFields, UpdateFIndex, Records, Options) -> end, AccIn, UpdateFIndex) end, [], NRecords) end), + %case BatchRemainSize of + % 0 -> [{Batch1Sql, lists:merge(RecBatchValues)}]; + % _ -> + % {Batch1Values, Batch2Values} = lists:split(BatchCount, RecBatchValues), + % case Batch1Sql of + % undefined -> [{Batch2Sql, lists:merge(Batch2Values)}]; + % _ -> [{Batch1Sql, lists:merge(Batch1Values)}, + % {Batch2Sql, lists:merge(Batch2Values)}] + % end + %end. case BatchRemainSize of 0 -> - [{Batch1Sql, FRecBatchValues} || FRecBatchValues <- lists:delete([], RecBatchValues)]; + % 当保存条数整除1000时，lists RecBatchValues最后的元素为[],要删除掉 + [{Batch1Sql, FRecBatchValues} || FRecBatchValues <- lists:delete([], RecBatchValues)]; + _ -> {Batch1Values, Batch2Values} = lists:split(BatchCount, RecBatchValues), case Batch1Sql of @@ -302,6 +317,7 @@ generate_update_sql(Table, UpdateFields, UpdateVals, {FieldPK, PKVal}) -> {Sql, lists:append(UpdateVals, [PKVal])}. +%generate_update_sql(Table, UpdateFields, UpdateVals, ) merge_rec(Rec1, Rec2, Fields, Options) -> IgnoreNil = proplists:get_value(ignore_nil, Options, false), From bd8689400974184bed436858724746f3e0dafc96 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Thu, 12 Feb 2015 10:59:34 +0800 Subject: [PATCH 52/61] insert on duplicate --- src/emysql_saver.erl | 47 ++++++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/src/emysql_saver.erl b/src/emysql_saver.erl index 3f07bf49..5f5cf528 100644 --- a/src/emysql_saver.erl +++ b/src/emysql_saver.erl @@ -28,6 +28,10 @@ %% %% or %% +%% Options = [{insert_ignore, true}, +%% {batch_size, 1000}, +%% {unique_fields, [out_id, login] } ] +%% emysql_saver:save(pool, test, ?INPUT(#test{data = "hello"}, Options). %% %% or %% @@ -255,19 +259,43 @@ generate_insert_sql(Table, UpdateFields, UpdateFIndex, Records, Options) -> BatchCount = length(Records) div BatchSize, BatchRemainSize = length(Records) rem BatchSize, + UniqueFields = proplists:get_value(unique_fields, Options, []), + Batch1Sql = case BatchCount of 0 -> undefined; _ -> SqlValues1 = string:join(lists:duplicate(BatchSize, ValuesInSql), ","), - string:join([SqlHead, "(", SqlFields, ") VALUES ", SqlValues1], " ") + Sql1 = string:join([SqlHead, "(", SqlFields, ") VALUES ", SqlValues1], " "), + case UniqueFields of + [] -> Sql1; + Fs1 when is_list(Fs1) -> + OnDupSubSql1 = lists:foldl( + fun(UniqueField, OnDupAcc) -> + UFieldStr = type_utils:any_to_list(UniqueField), + lists:append(UFieldStr ++ "= VALUES(" ++ UFieldStr ++ ")", OnDupAcc) + end, [], UniqueFields), + Sql1 ++ " ON DUPLICATE KEY UPDATE " ++ string:join(OnDupSubSql1, ", "); + _ -> Sql1 + end end, Batch2Sql = case BatchRemainSize of 0 -> undefined; _ -> SqlValues2 = string:join(lists:duplicate(BatchRemainSize, ValuesInSql), ","), - string:join([SqlHead, "(", SqlFields, ") VALUES ", SqlValues2], " ") + Sql2 = string:join([SqlHead, "(", SqlFields, ") VALUES ", SqlValues2], " "), + case UniqueFields of + [] -> Sql2; + Fs2 when is_list(Fs2) -> + OnDupSubSql2 = lists:foldl( + fun(UniqueField, OnDupAcc) -> + UFieldStr = type_utils:any_to_list(UniqueField), + lists:append(UFieldStr ++ "= VALUES(" ++ UFieldStr ++ ")", OnDupAcc) + end, [], UniqueFields), + Sql2 ++ " ON DUPLICATE KEY UPDATE " ++ string:join(OnDupSubSql2, ", "); + _ -> Sql2 + end end, - + RecBatchValues = lists_utils:split(BatchSize, Records, fun(NRecords) -> @@ -284,16 +312,7 @@ generate_insert_sql(Table, UpdateFields, UpdateFIndex, Records, Options) -> end, AccIn, UpdateFIndex) end, [], NRecords) end), - %case BatchRemainSize of - % 0 -> [{Batch1Sql, lists:merge(RecBatchValues)}]; - % _ -> - % {Batch1Values, Batch2Values} = lists:split(BatchCount, RecBatchValues), - % case Batch1Sql of - % undefined -> [{Batch2Sql, lists:merge(Batch2Values)}]; - % _ -> [{Batch1Sql, lists:merge(Batch1Values)}, - % {Batch2Sql, lists:merge(Batch2Values)}] - % end - %end. + case BatchRemainSize of 0 -> % 当保存条数整除1000时，lists RecBatchValues最后的元素为[],要删除掉 @@ -317,8 +336,6 @@ generate_update_sql(Table, UpdateFields, UpdateVals, {FieldPK, PKVal}) -> {Sql, lists:append(UpdateVals, [PKVal])}. -%generate_update_sql(Table, UpdateFields, UpdateVals, ) - merge_rec(Rec1, Rec2, Fields, Options) -> IgnoreNil = proplists:get_value(ignore_nil, Options, false), [_, RecChanged, MergedRec] = From fdce72bf7b0f02c6538e49be9b9703f29ce9d1a3 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Thu, 12 Feb 2015 17:25:33 +0800 Subject: [PATCH 53/61] compatible order in previous version --- src/emysql_query.erl | 22 ++++++++++++++++++---- src/emysql_saver.erl | 4 ++-- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/emysql_query.erl b/src/emysql_query.erl index 91c9fd72..5fe07d12 100644 --- a/src/emysql_query.erl +++ b/src/emysql_query.erl @@ -171,7 +171,7 @@ find_each(ConnOrPool, Table, SqlOptions, BatchSize, AsRec, Fun, AccIn) -> %%% Internal functions %%%=================================================================== build_sql(Table, SqlOptions) -> - {[FindSql, FindCondVals], _} = build_sql(Table, SqlOptions, undefined, undefined), + {[FindSql, FindCondVals], _, _} = build_sql(Table, SqlOptions, undefined, undefined), {FindSql, FindCondVals}. build_sql(Table, SqlOptions, BatchSize, BaseId) -> @@ -218,8 +218,17 @@ build_sql(Table, SqlOptions, BatchSize, BaseId) -> case proplists:get_value(order, SqlOptions) of undefined -> ["", id, "ASC"]; [OrderField, OrderSort] -> - ["ORDER BY " ++ type_utils:any_to_list(OrderField) ++ " " ++ type_utils:any_to_list(OrderSort), - OrderField, OrderSort] + OrderStr1 = "ORDER BY " ++ type_utils:any_to_list(OrderField) ++ " " ++ type_utils:any_to_list(OrderSort), + [OrderStr1, OrderField, OrderSort]; + [[OrderField, OrderSort] | _] = OrderOpt -> + MultiOrder = lists:foldl(fun([OField0, OSort0], OAcc) -> + lists:append(OAcc, [type_utils:any_to_list(OField0) ++ " " ++ type_utils:any_to_list(OSort0)]) + end, [], OrderOpt), + OrderStr2 = "ORDER BY " ++ string:join(MultiOrder, ", "), + [OrderStr2, OrderField, OrderSort]; %% TODO: should return multi-order + OrderBy -> + OrderStr3 = "ORDER BY " ++ type_utils:any_to_list(OrderBy), + [OrderStr3, undefined, undefined] end, {Where2, CondVals2} = @@ -260,7 +269,12 @@ do_find_each(ConnOrPool, Table, SqlOptions, BatchSize, [Rec, RecFields] = AsRec, {[Sql, CondVals], [_CountSql, _CountCondVals], [OrderField, _OSort]} = build_sql(Table, SqlOptions, BatchSize, BaseId), - do_find_each(ConnOrPool, Table, Sql, CondVals, BatchSize, AsRec, Fun, AccIn, Remain, OrderField, BaseId). + case OrderField of + undefined -> + throw(order_field_is_undefined); + _ -> + do_find_each(ConnOrPool, Table, Sql, CondVals, BatchSize, AsRec, Fun, AccIn, Remain, OrderField, BaseId) + end. do_find_each(ConnOrPool, Table, Sql, CondVals, BatchSize, [Rec, RecFields] = AsRec, Fun, AccIn, Remain, OrderField, BaseId) -> diff --git a/src/emysql_saver.erl b/src/emysql_saver.erl index 5f5cf528..4143a3be 100644 --- a/src/emysql_saver.erl +++ b/src/emysql_saver.erl @@ -272,7 +272,7 @@ generate_insert_sql(Table, UpdateFields, UpdateFIndex, Records, Options) -> OnDupSubSql1 = lists:foldl( fun(UniqueField, OnDupAcc) -> UFieldStr = type_utils:any_to_list(UniqueField), - lists:append(UFieldStr ++ "= VALUES(" ++ UFieldStr ++ ")", OnDupAcc) + lists:append(OnDupAcc, [UFieldStr ++ "= VALUES(" ++ UFieldStr ++ ")"]) end, [], UniqueFields), Sql1 ++ " ON DUPLICATE KEY UPDATE " ++ string:join(OnDupSubSql1, ", "); _ -> Sql1 @@ -289,7 +289,7 @@ generate_insert_sql(Table, UpdateFields, UpdateFIndex, Records, Options) -> OnDupSubSql2 = lists:foldl( fun(UniqueField, OnDupAcc) -> UFieldStr = type_utils:any_to_list(UniqueField), - lists:append(UFieldStr ++ "= VALUES(" ++ UFieldStr ++ ")", OnDupAcc) + lists:append(OnDupAcc, [UFieldStr ++ "= VALUES(" ++ UFieldStr ++ ")"]) end, [], UniqueFields), Sql2 ++ " ON DUPLICATE KEY UPDATE " ++ string:join(OnDupSubSql2, ", "); _ -> Sql2 From c4c71f38772f95facda496ca619b231e94bbb32a Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Thu, 12 Feb 2015 18:49:09 +0800 Subject: [PATCH 54/61] remove AccIn when it is undefined --- src/emysql_conv.erl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/emysql_conv.erl b/src/emysql_conv.erl index 0ad2b33a..00416eab 100644 --- a/src/emysql_conv.erl +++ b/src/emysql_conv.erl @@ -28,7 +28,7 @@ as_proplist(#result_packet{field_list=_Cols,rows=_Rows}) when is_list(_Cols), as_proplist(#result_packet{field_list=_Cols,rows=_Rows}) when is_list(_Cols), _Rows =:= [] -> []; -as_proplist(Res = #result_packet{field_list=Cols,rows=Rows}) when is_list(Cols), +as_proplist(#result_packet{field_list=Cols,rows=Rows} = Res) when is_list(Cols), is_list(Rows) -> Fields = emysql:field_names(Res), [begin @@ -36,7 +36,7 @@ as_proplist(Res = #result_packet{field_list=Cols,rows=Rows}) when is_list(Cols), end || R <- Rows]. %% @see emysql:as_record/1 -as_record(Result = #result_packet{}, RecordName, Fields, Fun, AccIn) when is_atom(RecordName), is_list(Fields), is_function(Fun) -> +as_record(#result_packet{} = Result, RecordName, Fields, Fun, AccIn) when is_atom(RecordName), is_list(Fields), is_function(Fun) -> Columns = Result#result_packet.field_list, S = lists:seq(1, length(Columns)), @@ -55,7 +55,7 @@ as_record(Result = #result_packet{}, RecordName, Fields, Fun, AccIn) when is_ato undefined -> F1 = fun(Row) -> RecordData = [ Fx(Row) || Fx <- Fs ], - Fun(list_to_tuple([RecordName|RecordData]), AccIn) + Fun(list_to_tuple([RecordName|RecordData])) end, [ F1(Row) || Row <- Result#result_packet.rows ]; _ -> @@ -67,11 +67,11 @@ as_record(Result = #result_packet{}, RecordName, Fields, Fun, AccIn) when is_ato end, {[], AccIn}, Result#result_packet.rows) end. -as_record(Result = #result_packet{}, RecordName, Fields, Fun) when is_atom(RecordName), is_list(Fields), is_function(Fun) -> - as_record(Result = #result_packet{}, RecordName, Fields, Fun, undefined). +as_record(#result_packet{} = Result, RecordName, Fields, Fun) when is_atom(RecordName), is_list(Fields), is_function(Fun) -> + as_record(Result, RecordName, Fields, Fun, undefined). -as_record(Result = #result_packet{}, RecordName, Fields) when is_atom(RecordName), is_list(Fields) -> +as_record(#result_packet{} = Result, RecordName, Fields) when is_atom(RecordName), is_list(Fields) -> as_record(Result, RecordName, Fields, fun(A) -> A end). %% @see emysql:as_json/1 From 161d213dacf6e4ccf38eed55e815292ae8e60cd4 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Fri, 13 Feb 2015 18:24:04 +0800 Subject: [PATCH 55/61] export as_record/5 --- src/emysql_conv.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/emysql_conv.erl b/src/emysql_conv.erl index 00416eab..460c025b 100644 --- a/src/emysql_conv.erl +++ b/src/emysql_conv.erl @@ -11,6 +11,7 @@ as_json/1, as_proplist/1, as_record/3, + as_record/5, as_record/4 ]). From 8548ae35f3c928767106f6287a07185fa4eecbb4 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Sun, 15 Feb 2015 16:50:01 +0800 Subject: [PATCH 56/61] fix bug in emysql_query:find_each --- src/emysql_query.erl | 77 ++++++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/src/emysql_query.erl b/src/emysql_query.erl index 5fe07d12..71dae461 100644 --- a/src/emysql_query.erl +++ b/src/emysql_query.erl @@ -273,45 +273,46 @@ do_find_each(ConnOrPool, Table, SqlOptions, BatchSize, [Rec, RecFields] = AsRec, undefined -> throw(order_field_is_undefined); _ -> - do_find_each(ConnOrPool, Table, Sql, CondVals, BatchSize, AsRec, Fun, AccIn, Remain, OrderField, BaseId) - end. - -do_find_each(ConnOrPool, Table, Sql, CondVals, BatchSize, [Rec, RecFields] = AsRec, - Fun, AccIn, Remain, OrderField, BaseId) -> - - Result = case ConnOrPool of - #emysql_connection{} = Conn -> - emysql_conn:execute(Conn, Sql, CondVals); - Pool -> - emysql:execute(Pool, Sql, CondVals) - end, - case Result of - #result_packet{rows = Rows} -> - case emysql_conv:as_record(Result, Rec, RecFields, Fun, AccIn) of - {[], NAccIn} -> NAccIn; - {Rs, NAccIn} -> - LastRow = lists_utils:last(Rs), - NextId = tuple_utils:value(OrderField, LastRow, RecFields), - case Remain - BatchSize > 0 of - true -> - do_find_each(ConnOrPool, Table, Sql, CondVals, BatchSize, AsRec, - Fun, NAccIn, (Remain - BatchSize), OrderField, NextId); - false -> - NAccIn - end; - Rs when is_list(Rs) -> - LastRow = lists_utils:last(Rs), - NextId = tuple_utils:value(OrderField, LastRow, RecFields), - case Remain - BatchSize > 0 of - true -> - do_find_each(ConnOrPool, Table, Sql, CondVals, BatchSize, AsRec, - Fun, undefined, (Remain - BatchSize), OrderField, NextId); - false -> + Result = case ConnOrPool of + #emysql_connection{} = Conn -> + emysql_conn:execute(Conn, Sql, CondVals); + Pool -> + emysql:execute(Pool, Sql, CondVals) + end, + case Result of + #result_packet{rows = Rows} -> + case emysql_conv:as_record(Result, Rec, RecFields, Fun, AccIn) of + {[], NAccIn} -> NAccIn; + {Rs, NAccIn} -> + LastRow = lists_utils:last(Rs), + NextId = tuple_utils:value(OrderField, LastRow, RecFields), + case Remain - BatchSize > 0 of + true -> + do_find_each(ConnOrPool, Table, SqlOptions, BatchSize, AsRec, + Fun, NAccIn, (Remain - BatchSize), NextId); + false -> + NAccIn + end; + Rs when is_list(Rs) -> + LastRow = lists_utils:last(Rs), + NextId = case erlang:is_record(LastRow, Rec) of + true -> + tuple_utils:value(OrderField, LastRow, RecFields); + _ -> + undefined + end, + case Remain - BatchSize > 0 of + true -> + do_find_each(ConnOrPool, Table, SqlOptions, BatchSize, AsRec, + Fun, undefined, (Remain - BatchSize), NextId); + false -> ok - end - end; - _ -> - AccIn + end + end; + _ -> + AccIn + end + end. From 9bed3ea33925b9ffb37246d8893cbd7eaa9d7ec2 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Sun, 15 Feb 2015 17:41:59 +0800 Subject: [PATCH 57/61] fix bug in lists:append --- src/emysql_conv.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emysql_conv.erl b/src/emysql_conv.erl index 460c025b..0cec7635 100644 --- a/src/emysql_conv.erl +++ b/src/emysql_conv.erl @@ -64,7 +64,7 @@ as_record(#result_packet{} = Result, RecordName, Fields, Fun, AccIn) when is_ato RecordData = [ Fx(Row) || Fx <- Fs ], Rec = list_to_tuple([RecordName|RecordData]), NFunAccIn = Fun(Rec, FunAccIn), - {[lists:append(Rec, RsAccIn), NFunAccIn]} + {[lists:append(RsAccIn, [Rec]), NFunAccIn]} end, {[], AccIn}, Result#result_packet.rows) end. From f29c4ebeab8d5363a10a0a0257a474f589bf762e Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Wed, 25 Feb 2015 17:15:20 +0800 Subject: [PATCH 58/61] fix bug in lists:foldl --- src/emysql_conv.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emysql_conv.erl b/src/emysql_conv.erl index 0cec7635..2d4bd556 100644 --- a/src/emysql_conv.erl +++ b/src/emysql_conv.erl @@ -64,7 +64,7 @@ as_record(#result_packet{} = Result, RecordName, Fields, Fun, AccIn) when is_ato RecordData = [ Fx(Row) || Fx <- Fs ], Rec = list_to_tuple([RecordName|RecordData]), NFunAccIn = Fun(Rec, FunAccIn), - {[lists:append(RsAccIn, [Rec]), NFunAccIn]} + {lists:append(RsAccIn, [Rec]), NFunAccIn} end, {[], AccIn}, Result#result_packet.rows) end. From 38f562d0f3050266f2515551d2018620c5c3d356 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Mon, 11 May 2015 18:43:02 +0800 Subject: [PATCH 59/61] Fix in statement in emysql_query --- src/emysql_query.erl | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/src/emysql_query.erl b/src/emysql_query.erl index 71dae461..20d99a8b 100644 --- a/src/emysql_query.erl +++ b/src/emysql_query.erl @@ -23,7 +23,7 @@ %%-------------------------------------------------------------------- %% @doc %% e.g: find(my_pool, ["SELECT name FROM users where age > ? and time > ?", 12, "2013-12-03" ]) -%% +%% %% %% @spec %% @end @@ -73,12 +73,12 @@ find(ConnOrPool, Table, SqlOptions, ?AS_VAL) -> #result_packet{rows = [[R] ]} -> R; #result_packet{rows = Rs } -> lists:foldl(fun([R], AccIn) -> - lists:append(AccIn, [R]) + lists:append(AccIn, [R]) end, [], Rs); #error_packet{code = Code, msg = Msg} -> throw({Code, Msg}) end; - + find(ConnOrPool, Table, SqlOptions, [Rec, RecFields] = _AsRec) -> Result = find(ConnOrPool, Table, SqlOptions), emysql_conv:as_record(Result, Rec, RecFields). @@ -134,11 +134,11 @@ find_each(ConnOrPool, Table, SqlOptions, AsRec, Fun, AccIn) -> find_each(ConnOrPool, Table, SqlOptions, BatchSize, AsRec, Fun, AccIn) -> BaseId = undefined, - + {[FindSql, FindCondVals], [CountSql, CountCondVals], [OField, OSort]} = build_sql(Table, SqlOptions, BatchSize, BaseId), - + Result = case ConnOrPool of #emysql_connection{} = Conn -> emysql_conn:execute(Conn, CountSql, CountCondVals); @@ -165,8 +165,8 @@ find_each(ConnOrPool, Table, SqlOptions, BatchSize, AsRec, Fun, AccIn) -> _ -> ok end. - - + + %%%=================================================================== %%% Internal functions %%%=================================================================== @@ -181,7 +181,7 @@ build_sql(Table, SqlOptions, BatchSize, BaseId) -> V when is_list(V) -> string:join(V, ","); _ -> throw(bad_sql_select) end, - + {Where, CondVals} = case proplists:get_value(where, SqlOptions) of undefined -> {"", [ ]}; [ ] -> {"", [ ]}; @@ -194,9 +194,13 @@ build_sql(Table, SqlOptions, BatchSize, BaseId) -> S = type_utils:any_to_list(K) ++ " BETWEEN ? AND ? ", {[S | StmtAcc], [V1, V2 | ValAcc]}; {K, in, V3} when is_list(V3) -> - S = type_utils:any_to_list(K) ++ " IN (?) ", - NV3 = string:join(V3, ", "), - {[S | StmtAcc], [NV3 | ValAcc]}; + InStmt = string:join(lists:duplicate(length(V3), "?"), ","), + S = type_utils:any_to_list(K) ++ " IN (" ++ InStmt ++ ") ", + + NV3 = lists:foldl(fun(V3Item, ValAcc2) -> + [V3Item | ValAcc2] + end, ValAcc, V3), + {[S | StmtAcc], NV3 }; {K, OP, V4} -> S = string:join([type_utils:any_to_list(K), type_utils:any_to_list(OP), @@ -213,7 +217,7 @@ build_sql(Table, SqlOptions, BatchSize, BaseId) -> [ Cond ] -> {"WHERE " ++ Cond, [ ]}; [Cond | Values] -> {"WHERE " ++ Cond, Values} end, - + [Order, OField, OSort] = case proplists:get_value(order, SqlOptions) of undefined -> ["", id, "ASC"]; @@ -230,7 +234,7 @@ build_sql(Table, SqlOptions, BatchSize, BaseId) -> OrderStr3 = "ORDER BY " ++ type_utils:any_to_list(OrderBy), [OrderStr3, undefined, undefined] end, - + {Where2, CondVals2} = case {BaseId, OSort, Where} of {infinate, _, _} -> {Where, CondVals}; @@ -244,10 +248,10 @@ build_sql(Table, SqlOptions, BatchSize, BaseId) -> {Where ++ " AND " ++ type_utils:any_to_list(OField) ++ " < ?", lists:append(CondVals, [BaseId])} end, - - Limit = + + Limit = case BatchSize of - undefined -> + undefined -> case proplists:get_value(limit, SqlOptions) of undefined -> ""; [LV1, LV2] -> @@ -312,7 +316,5 @@ do_find_each(ConnOrPool, Table, SqlOptions, BatchSize, [Rec, RecFields] = AsRec, _ -> AccIn end - + end. - - From a930662d655cb4b1ba83ea9b38ccdf9032abfdd9 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Fri, 11 Dec 2015 16:06:43 +0800 Subject: [PATCH 60/61] Remove deprecated api --- include/emysql.hrl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/emysql.hrl b/include/emysql.hrl index 38edd472..f1fb97e2 100644 --- a/include/emysql.hrl +++ b/include/emysql.hrl @@ -34,9 +34,9 @@ port :: number(), database :: string(), encoding :: utf8 | latin1 | {utf8, utf8_unicode_ci} | {utf8, utf8_general_ci}, - available=queue:new() :: queue(), - locked=gb_trees:empty() :: gb_tree(), - waiting=queue:new() :: queue(), + available=queue:new() :: queue:queue(), + locked=gb_trees:empty() :: gb_tree:tree(), + waiting=queue:new() :: queue:queue(), start_cmds=[] :: string(), conn_test_period=0 :: number(), connect_timeout=infinity :: number() | infinity, From f1ab3f80fc81834272a998d709023d91b76de801 Mon Sep 17 00:00:00 2001 From: Jack Tang Date: Fri, 11 Dec 2015 16:11:12 +0800 Subject: [PATCH 61/61] Remove deprecated api --- src/emysql.erl | 2 +- src/emysql_conn_mgr.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/emysql.erl b/src/emysql.erl index 17f889ba..50b699ab 100644 --- a/src/emysql.erl +++ b/src/emysql.erl @@ -681,7 +681,7 @@ result_type(#eof_packet{}) -> eof. -spec as_dict(Result) -> Dict when Result :: #result_packet{}, - Dict :: dict(). + Dict :: dict:dict(). as_dict(Res) -> emysql_conv:as_dict(Res). diff --git a/src/emysql_conn_mgr.erl b/src/emysql_conn_mgr.erl index b14a08d6..97536a06 100644 --- a/src/emysql_conn_mgr.erl +++ b/src/emysql_conn_mgr.erl @@ -40,7 +40,7 @@ -include("emysql.hrl"). --record(state, {pools, lockers = dict:new() :: dict()}). +-record(state, {pools, lockers = dict:new() :: dict:dict()}). %%==================================================================== %% API